From 7aea6aa210a8939ac208fb7540d1b46ba69a995f Mon Sep 17 00:00:00 2001
From: Alan Pearce
Date: Wed, 15 Jan 2025 22:25:33 +0100
Subject: feat: enable searching via program names for multi-program packages
implements: https://todo.sr.ht/~alanpearce/searchix/6
---
internal/components/packageDetail.templ | 16 +++++-
internal/config/default.go | 6 +-
internal/config/structs.go | 6 ++
internal/importer/main.go | 13 +++++
internal/importer/package.go | 47 ++++++++++++----
internal/index/index_meta.go | 9 +--
internal/index/indexer.go | 1 +
internal/index/search.go | 5 ++
internal/nix/package.go | 1 +
internal/programs/programs.go | 97 +++++++++++++++++++++++++++++++++
10 files changed, 182 insertions(+), 19 deletions(-)
create mode 100644 internal/programs/programs.go
(limited to 'internal')
diff --git a/internal/components/packageDetail.templ b/internal/components/packageDetail.templ
index 65c74aa..84d2bdf 100644
--- a/internal/components/packageDetail.templ
+++ b/internal/components/packageDetail.templ
@@ -1,8 +1,6 @@
package components
-import (
- "go.alanpearce.eu/searchix/internal/nix"
-)
+import "go.alanpearce.eu/searchix/internal/nix"
func licenseName(l nix.License) string {
if l.FullName != "" {
@@ -32,6 +30,18 @@ templ PackageDetail(pkg nix.Package) {
{ pkg.MainProgram }
}
+ if len(pkg.Programs) > 0 {
+ Programs
+
+
+ for _, p := range pkg.Programs {
+ -
+
{ p }
+
+ }
+
+
+ }
if len(pkg.Homepages) > 0 {
Homepage
diff --git a/internal/config/default.go b/internal/config/default.go
index a857799..5202678 100644
--- a/internal/config/default.go
+++ b/internal/config/default.go
@@ -113,8 +113,12 @@ var DefaultConfig = Config{
Fetcher: ChannelNixpkgs,
Channel: "nixos-unstable",
OutputPath: "packages.json.br",
- Timeout: Duration{5 * time.Minute},
+ Timeout: Duration{15 * time.Minute},
Repo: nixpkgs,
+ Programs: ProgramsDB{
+ Enable: true,
+ Attribute: "programs.sqlite",
+ },
},
},
},
diff --git a/internal/config/structs.go b/internal/config/structs.go
index b31e0cd..e73425b 100644
--- a/internal/config/structs.go
+++ b/internal/config/structs.go
@@ -48,6 +48,12 @@ type Source struct {
Timeout Duration `comment:"Abort import if it takes longer than this."`
OutputPath string `comment:"(Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json."`
Repo Repository `comment:"Used to generate declaration/definition links"`
+ Programs ProgramsDB `comment:"Used to enable searching for programs in multi-program packages"`
+}
+
+type ProgramsDB struct {
+ Enable bool `comment:"Enable searching for programs in multi-program packages"`
+ Attribute string `comment:"Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file"`
}
func (source *Source) String() string {
diff --git a/internal/importer/main.go b/internal/importer/main.go
index 4c66501..dfa2477 100644
--- a/internal/importer/main.go
+++ b/internal/importer/main.go
@@ -11,6 +11,7 @@ import (
"go.alanpearce.eu/searchix/internal/config"
"go.alanpearce.eu/searchix/internal/fetcher"
"go.alanpearce.eu/searchix/internal/index"
+ "go.alanpearce.eu/searchix/internal/programs"
"go.alanpearce.eu/x/log"
"github.com/pkg/errors"
@@ -78,6 +79,17 @@ func createSourceImporter(
)
if sourceMeta.Updated.After(previousUpdate) || forceUpdate {
+ var pdb *programs.DB
+
+ if source.Programs.Enable {
+ pdb, err = programs.Instantiate(ctx, source, log.Named("programs"))
+ if err != nil {
+ logger.Warn("programs database instantiation failed", "error", err)
+ }
+ if pdb.Path != sourceMeta.ProgramsPath {
+ sourceMeta.ProgramsPath = pdb.Path
+ }
+ }
err = setRepoRevision(files.Revision, source)
if err != nil {
@@ -105,6 +117,7 @@ func createSourceImporter(
files.Packages,
source,
logger.Named("processor"),
+ pdb,
)
}
if err != nil {
diff --git a/internal/importer/package.go b/internal/importer/package.go
index 80adc38..59bccd8 100644
--- a/internal/importer/package.go
+++ b/internal/importer/package.go
@@ -9,6 +9,7 @@ import (
"go.alanpearce.eu/searchix/internal/config"
"go.alanpearce.eu/searchix/internal/nix"
+ "go.alanpearce.eu/searchix/internal/programs"
"go.alanpearce.eu/x/log"
"github.com/bcicen/jstream"
@@ -39,12 +40,13 @@ type maintainerJSON struct {
}
type PackageIngester struct {
- dec *jstream.Decoder
- ms *mapstructure.Decoder
- log *log.Logger
- pkg packageJSON
- infile io.ReadCloser
- source *config.Source
+ dec *jstream.Decoder
+ ms *mapstructure.Decoder
+ log *log.Logger
+ pkg packageJSON
+ infile io.ReadCloser
+ source *config.Source
+ programs *programs.DB
}
func makeAdhocLicense(name string) nix.License {
@@ -66,13 +68,15 @@ func NewPackageProcessor(
infile io.ReadCloser,
source *config.Source,
log *log.Logger,
+ programsDB *programs.DB,
) (*PackageIngester, error) {
i := &PackageIngester{
- dec: jstream.NewDecoder(infile, 2).EmitKV(),
- log: log,
- pkg: packageJSON{},
- infile: infile,
- source: source,
+ dec: jstream.NewDecoder(infile, 2).EmitKV(),
+ log: log,
+ pkg: packageJSON{},
+ infile: infile,
+ source: source,
+ programs: programsDB,
}
ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
@@ -116,6 +120,14 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, <
results := make(chan nix.Importable)
errs := make(chan error)
+ if i.programs != nil {
+ err := i.programs.Open()
+ if err != nil {
+ errs <- errors.WithMessage(err, "could not open programs database")
+ i.programs = nil
+ }
+ }
+
go func() {
defer i.infile.Close()
defer close(results)
@@ -124,6 +136,7 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, <
outer:
for mv := range i.dec.Stream() {
var err error
+ var programs []string
select {
case <-ctx.Done():
break outer
@@ -216,6 +229,13 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, <
continue
}
+ if i.programs != nil {
+ programs, err = i.programs.GetPackagePrograms(ctx, kv.Key)
+ if err != nil {
+ errs <- errors.WithMessagef(err, "failed to get programs for package %s", i.pkg.Name)
+ }
+ }
+
maintainers := make([]nix.Maintainer, len(i.pkg.Meta.Maintainers))
for i, m := range i.pkg.Meta.Maintainers {
maintainers[i] = nix.Maintainer{
@@ -250,8 +270,13 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, <
Licenses: licenses,
Maintainers: maintainers,
Definition: url,
+ Programs: programs,
}
}
+
+ if i.programs != nil {
+ i.programs.Close()
+ }
}()
return results, errs
diff --git a/internal/index/index_meta.go b/internal/index/index_meta.go
index e67c6f2..7d133cd 100644
--- a/internal/index/index_meta.go
+++ b/internal/index/index_meta.go
@@ -11,12 +11,13 @@ import (
"github.com/pkg/errors"
)
-const CurrentSchemaVersion = 2
+const CurrentSchemaVersion = 3
type SourceMeta struct {
- Updated time.Time
- Path string
- Rev string
+ Updated time.Time
+ Path string
+ Rev string
+ ProgramsPath string
}
type data struct {
diff --git a/internal/index/indexer.go b/internal/index/indexer.go
index 0c12104..9c291b8 100644
--- a/internal/index/indexer.go
+++ b/internal/index/indexer.go
@@ -126,6 +126,7 @@ func createIndexMapping() (mapping.IndexMapping, error) {
packageMapping.AddFieldMappingsAt("MainProgram", keywordFieldMapping)
packageMapping.AddFieldMappingsAt("PackageSet", keywordFieldMapping)
packageMapping.AddFieldMappingsAt("Platforms", keywordFieldMapping)
+ packageMapping.AddFieldMappingsAt("Programs", keywordFieldMapping)
indexMapping.AddDocumentMapping("option", optionMapping)
indexMapping.AddDocumentMapping("package", packageMapping)
diff --git a/internal/index/search.go b/internal/index/search.go
index 9d77488..a62f484 100644
--- a/internal/index/search.go
+++ b/internal/index/search.go
@@ -164,6 +164,11 @@ func (index *ReadIndex) Search(
query.AddShould(q)
}
+ programsQuery := bleve.NewMatchQuery(keyword)
+ programsQuery.SetField("Programs")
+ programsQuery.Analyzer = "keyword_single"
+ query.AddShould(programsQuery)
+
attrQuery := bleve.NewMatchQuery(keyword)
attrQuery.SetField("Attribute")
attrQuery.Analyzer = "keyword_single"
diff --git a/internal/nix/package.go b/internal/nix/package.go
index 42bf77a..0b6519e 100644
--- a/internal/nix/package.go
+++ b/internal/nix/package.go
@@ -11,6 +11,7 @@ type Package struct {
Licenses []License
LongDescription Markdown
MainProgram string
+ Programs []string
Maintainers []Maintainer
PackageSet string
Platforms []string
diff --git a/internal/programs/programs.go b/internal/programs/programs.go
new file mode 100644
index 0000000..1dbfff7
--- /dev/null
+++ b/internal/programs/programs.go
@@ -0,0 +1,97 @@
+package programs
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "os/exec"
+ "strings"
+
+ "github.com/pkg/errors"
+ "go.alanpearce.eu/searchix/internal/config"
+ "go.alanpearce.eu/x/log"
+ _ "modernc.org/sqlite" //nolint:blank-imports // sqlite driver needed for database/sql
+)
+
+type DB struct {
+ Path string
+ Source *config.Source
+
+ logger *log.Logger
+ db *sql.DB
+}
+
+func Instantiate(ctx context.Context, source *config.Source, logger *log.Logger) (*DB, error) {
+ // nix-instantiate --eval --json -I nixpkgs=channel:nixos-unstable --expr 'toString '
+ args := []string{
+ "--eval",
+ "--json",
+ "-I", fmt.Sprintf("%s=channel:%s", source.Key, source.Channel),
+ "--expr", fmt.Sprintf("toString <%s/%s>", source.Key, source.Programs.Attribute),
+ }
+
+ logger.Debug("nix-instantiate command", "args", args)
+ cmd := exec.CommandContext(ctx, "nix-instantiate", args...)
+ out, err := cmd.Output()
+ if err != nil {
+ return nil, errors.WithMessage(err, "failed to run nix-instantiate")
+ }
+
+ outPath := strings.Trim(strings.TrimSpace(string(out)), "\"")
+ logger.Debug("got output path", "outputPath", outPath)
+
+ return &DB{
+ Source: source,
+ Path: outPath,
+
+ logger: logger,
+ }, nil
+}
+
+func (p *DB) Open() error {
+ db, err := sql.Open("sqlite", p.Path)
+ if err != nil {
+ return errors.WithMessage(err, "failed to open sqlite database")
+ }
+ p.db = db
+
+ return nil
+}
+
+func (p *DB) Close() error {
+ if err := p.db.Close(); err != nil {
+ return errors.WithMessage(err, "failed to close sqlite database")
+ }
+
+ return nil
+}
+
+func (p *DB) GetPackagePrograms(ctx context.Context, pkg string) (programs []string, err error) {
+ if p.db == nil {
+ return nil, errors.New("database not open")
+ }
+ rows, err := p.db.QueryContext(ctx, `
+SELECT name
+FROM Programs
+WHERE package = ?
+GROUP BY name, package`, pkg)
+ if err != nil {
+ return nil, errors.WithMessage(err, "failed to execute query")
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var name string
+ if err := rows.Scan(&name); err != nil {
+ return nil, errors.WithMessage(err, "failed to scan row")
+ }
+
+ programs = append(programs, name)
+ }
+ rerr := rows.Close()
+ if rerr != nil {
+ return nil, errors.WithMessage(rerr, "sql error")
+ }
+
+ return
+}
--
cgit 1.4.1