feat: enable searching via program names for multi-program packages

implements: https://todo.sr.ht/~alanpearce/searchix/6

10 files changed, 182 insertions, 19 deletions

diff --git a/internal/components/packageDetail.templ b/internal/components/packageDetail.templ
index 65c74aa..84d2bdf 100644
--- a/internal/components/packageDetail.templ
+++ b/internal/components/packageDetail.templ
@@ -1,8 +1,6 @@
 package components
 
-import (
-	"go.alanpearce.eu/searchix/internal/nix"
-)
+import "go.alanpearce.eu/searchix/internal/nix"
 
 func licenseName(l nix.License) string {
 	if l.FullName != "" {
@@ -32,6 +30,18 @@ templ PackageDetail(pkg nix.Package) {
 				<code>{ pkg.MainProgram }</code>
 			</dd>
 		}
+		if len(pkg.Programs) > 0 {
+			<dt>Programs</dt>
+			<dd>
+				<ul>
+					for _, p := range pkg.Programs {
+						<li>
+							<code>{ p }</code>
+						</li>
+					}
+				</ul>
+			</dd>
+		}
 		if len(pkg.Homepages) > 0 {
 			<dt>Homepage</dt>
 			<dd>
diff --git a/internal/config/default.go b/internal/config/default.go
index a857799..5202678 100644
--- a/internal/config/default.go
+++ b/internal/config/default.go
@@ -113,8 +113,12 @@ var DefaultConfig = Config{
 				Fetcher:    ChannelNixpkgs,
 				Channel:    "nixos-unstable",
 				OutputPath: "packages.json.br",
-				Timeout:    Duration{5 * time.Minute},
+				Timeout:    Duration{15 * time.Minute},
 				Repo:       nixpkgs,
+				Programs: ProgramsDB{
+					Enable:    true,
+					Attribute: "programs.sqlite",
+				},
 			},
 		},
 	},
diff --git a/internal/config/structs.go b/internal/config/structs.go
index b31e0cd..e73425b 100644
--- a/internal/config/structs.go
+++ b/internal/config/structs.go
@@ -48,6 +48,12 @@ type Source struct {
 	Timeout    Duration     `comment:"Abort import if it takes longer than this."`
 	OutputPath string       `comment:"(Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json."`
 	Repo       Repository   `comment:"Used to generate declaration/definition links"`
+	Programs   ProgramsDB   `comment:"Used to enable searching for programs in multi-program packages"`
+}
+
+type ProgramsDB struct {
+	Enable    bool   `comment:"Enable searching for programs in multi-program packages"`
+	Attribute string `comment:"Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file"`
 }
 
 func (source *Source) String() string {
diff --git a/internal/importer/main.go b/internal/importer/main.go
index 4c66501..dfa2477 100644
--- a/internal/importer/main.go
+++ b/internal/importer/main.go
@@ -11,6 +11,7 @@ import (
 	"go.alanpearce.eu/searchix/internal/config"
 	"go.alanpearce.eu/searchix/internal/fetcher"
 	"go.alanpearce.eu/searchix/internal/index"
+	"go.alanpearce.eu/searchix/internal/programs"
 	"go.alanpearce.eu/x/log"
 
 	"github.com/pkg/errors"
@@ -78,6 +79,17 @@ func createSourceImporter(
 		)
 
 		if sourceMeta.Updated.After(previousUpdate) || forceUpdate {
+			var pdb *programs.DB
+
+			if source.Programs.Enable {
+				pdb, err = programs.Instantiate(ctx, source, log.Named("programs"))
+				if err != nil {
+					logger.Warn("programs database instantiation failed", "error", err)
+				}
+				if pdb.Path != sourceMeta.ProgramsPath {
+					sourceMeta.ProgramsPath = pdb.Path
+				}
+			}
 
 			err = setRepoRevision(files.Revision, source)
 			if err != nil {
@@ -105,6 +117,7 @@ func createSourceImporter(
 					files.Packages,
 					source,
 					logger.Named("processor"),
+					pdb,
 				)
 			}
 			if err != nil {
diff --git a/internal/importer/package.go b/internal/importer/package.go
index 80adc38..59bccd8 100644
--- a/internal/importer/package.go
+++ b/internal/importer/package.go
@@ -9,6 +9,7 @@ import (
 
 	"go.alanpearce.eu/searchix/internal/config"
 	"go.alanpearce.eu/searchix/internal/nix"
+	"go.alanpearce.eu/searchix/internal/programs"
 	"go.alanpearce.eu/x/log"
 
 	"github.com/bcicen/jstream"
@@ -39,12 +40,13 @@ type maintainerJSON struct {
 }
 
 type PackageIngester struct {
-	dec    *jstream.Decoder
-	ms     *mapstructure.Decoder
-	log    *log.Logger
-	pkg    packageJSON
-	infile io.ReadCloser
-	source *config.Source
+	dec      *jstream.Decoder
+	ms       *mapstructure.Decoder
+	log      *log.Logger
+	pkg      packageJSON
+	infile   io.ReadCloser
+	source   *config.Source
+	programs *programs.DB
 }
 
 func makeAdhocLicense(name string) nix.License {
@@ -66,13 +68,15 @@ func NewPackageProcessor(
 	infile io.ReadCloser,
 	source *config.Source,
 	log *log.Logger,
+	programsDB *programs.DB,
 ) (*PackageIngester, error) {
 	i := &PackageIngester{
-		dec:    jstream.NewDecoder(infile, 2).EmitKV(),
-		log:    log,
-		pkg:    packageJSON{},
-		infile: infile,
-		source: source,
+		dec:      jstream.NewDecoder(infile, 2).EmitKV(),
+		log:      log,
+		pkg:      packageJSON{},
+		infile:   infile,
+		source:   source,
+		programs: programsDB,
 	}
 
 	ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
@@ -116,6 +120,14 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, <
 	results := make(chan nix.Importable)
 	errs := make(chan error)
 
+	if i.programs != nil {
+		err := i.programs.Open()
+		if err != nil {
+			errs <- errors.WithMessage(err, "could not open programs database")
+			i.programs = nil
+		}
+	}
+
 	go func() {
 		defer i.infile.Close()
 		defer close(results)
@@ -124,6 +136,7 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, <
 	outer:
 		for mv := range i.dec.Stream() {
 			var err error
+			var programs []string
 			select {
 			case <-ctx.Done():
 				break outer
@@ -216,6 +229,13 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, <
 				continue
 			}
 
+			if i.programs != nil {
+				programs, err = i.programs.GetPackagePrograms(ctx, kv.Key)
+				if err != nil {
+					errs <- errors.WithMessagef(err, "failed to get programs for package %s", i.pkg.Name)
+				}
+			}
+
 			maintainers := make([]nix.Maintainer, len(i.pkg.Meta.Maintainers))
 			for i, m := range i.pkg.Meta.Maintainers {
 				maintainers[i] = nix.Maintainer{
@@ -250,8 +270,13 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, <
 				Licenses:        licenses,
 				Maintainers:     maintainers,
 				Definition:      url,
+				Programs:        programs,
 			}
 		}
+
+		if i.programs != nil {
+			i.programs.Close()
+		}
 	}()
 
 	return results, errs
diff --git a/internal/index/index_meta.go b/internal/index/index_meta.go
index e67c6f2..7d133cd 100644
--- a/internal/index/index_meta.go
+++ b/internal/index/index_meta.go
@@ -11,12 +11,13 @@ import (
 	"github.com/pkg/errors"
 )
 
-const CurrentSchemaVersion = 2
+const CurrentSchemaVersion = 3
 
 type SourceMeta struct {
-	Updated time.Time
-	Path    string
-	Rev     string
+	Updated      time.Time
+	Path         string
+	Rev          string
+	ProgramsPath string
 }
 
 type data struct {
diff --git a/internal/index/indexer.go b/internal/index/indexer.go
index 0c12104..9c291b8 100644
--- a/internal/index/indexer.go
+++ b/internal/index/indexer.go
@@ -126,6 +126,7 @@ func createIndexMapping() (mapping.IndexMapping, error) {
 	packageMapping.AddFieldMappingsAt("MainProgram", keywordFieldMapping)
 	packageMapping.AddFieldMappingsAt("PackageSet", keywordFieldMapping)
 	packageMapping.AddFieldMappingsAt("Platforms", keywordFieldMapping)
+	packageMapping.AddFieldMappingsAt("Programs", keywordFieldMapping)
 
 	indexMapping.AddDocumentMapping("option", optionMapping)
 	indexMapping.AddDocumentMapping("package", packageMapping)
diff --git a/internal/index/search.go b/internal/index/search.go
index 9d77488..a62f484 100644
--- a/internal/index/search.go
+++ b/internal/index/search.go
@@ -164,6 +164,11 @@ func (index *ReadIndex) Search(
 		query.AddShould(q)
 	}
 
+	programsQuery := bleve.NewMatchQuery(keyword)
+	programsQuery.SetField("Programs")
+	programsQuery.Analyzer = "keyword_single"
+	query.AddShould(programsQuery)
+
 	attrQuery := bleve.NewMatchQuery(keyword)
 	attrQuery.SetField("Attribute")
 	attrQuery.Analyzer = "keyword_single"
diff --git a/internal/nix/package.go b/internal/nix/package.go
index 42bf77a..0b6519e 100644
--- a/internal/nix/package.go
+++ b/internal/nix/package.go
@@ -11,6 +11,7 @@ type Package struct {
 	Licenses        []License
 	LongDescription Markdown
 	MainProgram     string
+	Programs        []string
 	Maintainers     []Maintainer
 	PackageSet      string
 	Platforms       []string
diff --git a/internal/programs/programs.go b/internal/programs/programs.go
new file mode 100644
index 0000000..1dbfff7
--- /dev/null
+++ b/internal/programs/programs.go
@@ -0,0 +1,97 @@
+package programs
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"github.com/pkg/errors"
+	"go.alanpearce.eu/searchix/internal/config"
+	"go.alanpearce.eu/x/log"
+	_ "modernc.org/sqlite" //nolint:blank-imports // sqlite driver needed for database/sql
+)
+
+type DB struct {
+	Path   string
+	Source *config.Source
+
+	logger *log.Logger
+	db     *sql.DB
+}
+
+func Instantiate(ctx context.Context, source *config.Source, logger *log.Logger) (*DB, error) {
+	// nix-instantiate --eval --json -I nixpkgs=channel:nixos-unstable --expr 'toString <nixpkgs/programs.sqlite>'
+	args := []string{
+		"--eval",
+		"--json",
+		"-I", fmt.Sprintf("%s=channel:%s", source.Key, source.Channel),
+		"--expr", fmt.Sprintf("toString <%s/%s>", source.Key, source.Programs.Attribute),
+	}
+
+	logger.Debug("nix-instantiate command", "args", args)
+	cmd := exec.CommandContext(ctx, "nix-instantiate", args...)
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to run nix-instantiate")
+	}
+
+	outPath := strings.Trim(strings.TrimSpace(string(out)), "\"")
+	logger.Debug("got output path", "outputPath", outPath)
+
+	return &DB{
+		Source: source,
+		Path:   outPath,
+
+		logger: logger,
+	}, nil
+}
+
+func (p *DB) Open() error {
+	db, err := sql.Open("sqlite", p.Path)
+	if err != nil {
+		return errors.WithMessage(err, "failed to open sqlite database")
+	}
+	p.db = db
+
+	return nil
+}
+
+func (p *DB) Close() error {
+	if err := p.db.Close(); err != nil {
+		return errors.WithMessage(err, "failed to close sqlite database")
+	}
+
+	return nil
+}
+
+func (p *DB) GetPackagePrograms(ctx context.Context, pkg string) (programs []string, err error) {
+	if p.db == nil {
+		return nil, errors.New("database not open")
+	}
+	rows, err := p.db.QueryContext(ctx, `
+SELECT name
+FROM Programs
+WHERE package = ?
+GROUP BY name, package`, pkg)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to execute query")
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var name string
+		if err := rows.Scan(&name); err != nil {
+			return nil, errors.WithMessage(err, "failed to scan row")
+		}
+
+		programs = append(programs, name)
+	}
+	rerr := rows.Close()
+	if rerr != nil {
+		return nil, errors.WithMessage(rerr, "sql error")
+	}
+
+	return
+}