diff options
author | Alan Pearce | 2025-01-15 22:25:33 +0100 |
---|---|---|
committer | Alan Pearce | 2025-01-15 22:25:33 +0100 |
commit | 7aea6aa210a8939ac208fb7540d1b46ba69a995f (patch) | |
tree | 80f8db2539289ca545eb356bf87e2b764d39c966 | |
parent | b26ddba432f8bde78022d2fc8837f0ffb25448b1 (diff) | |
download | searchix-7aea6aa210a8939ac208fb7540d1b46ba69a995f.tar.lz searchix-7aea6aa210a8939ac208fb7540d1b46ba69a995f.tar.zst searchix-7aea6aa210a8939ac208fb7540d1b46ba69a995f.zip |
feat: enable searching via program names for multi-program packages
implements: https://todo.sr.ht/~alanpearce/searchix/6
-rw-r--r-- | defaults.toml | 30 | ||||
-rw-r--r-- | go.mod | 13 | ||||
-rw-r--r-- | go.sum | 45 | ||||
-rw-r--r-- | gomod2nix.toml | 39 | ||||
-rw-r--r-- | internal/components/packageDetail.templ | 16 | ||||
-rw-r--r-- | internal/config/default.go | 6 | ||||
-rw-r--r-- | internal/config/structs.go | 6 | ||||
-rw-r--r-- | internal/importer/main.go | 13 | ||||
-rw-r--r-- | internal/importer/package.go | 47 | ||||
-rw-r--r-- | internal/index/index_meta.go | 9 | ||||
-rw-r--r-- | internal/index/indexer.go | 1 | ||||
-rw-r--r-- | internal/index/search.go | 5 | ||||
-rw-r--r-- | internal/nix/package.go | 1 | ||||
-rw-r--r-- | internal/programs/programs.go | 97 |
14 files changed, 308 insertions, 20 deletions
diff --git a/defaults.toml b/defaults.toml index 4f95929..bfd2efa 100644 --- a/defaults.toml +++ b/defaults.toml @@ -101,6 +101,13 @@ Type = 'github' Owner = 'LnL7' Repo = 'nix-darwin' +# Used to enable searching for programs in multi-program packages +[Importer.Sources.darwin.Programs] +# Enable searching for programs in multi-program packages +Enable = false +# Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file +Attribute = '' + [Importer.Sources.home-manager] # Human-readable name of source for generating links Name = 'Home Manager' @@ -134,6 +141,13 @@ Type = 'github' Owner = 'nix-community' Repo = 'home-manager' +# Used to enable searching for programs in multi-program packages +[Importer.Sources.home-manager.Programs] +# Enable searching for programs in multi-program packages +Enable = false +# Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file +Attribute = '' + [Importer.Sources.nixos] # Human-readable name of source for generating links Name = 'NixOS' @@ -167,6 +181,13 @@ Type = 'github' Owner = 'NixOS' Repo = 'nixpkgs' +# Used to enable searching for programs in multi-program packages +[Importer.Sources.nixos.Programs] +# Enable searching for programs in multi-program packages +Enable = false +# Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file +Attribute = '' + [Importer.Sources.nixpkgs] # Human-readable name of source for generating links Name = 'Nix Packages' @@ -189,7 +210,7 @@ Attribute = '' # (Fetcher=channel) Sub-path of imported channel which contains the attribute above, e.g. release.nix ImportPath = '' # Abort import if it takes longer than this. -Timeout = '5m0s' +Timeout = '15m0s' # (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json. OutputPath = 'packages.json.br' @@ -199,3 +220,10 @@ OutputPath = 'packages.json.br' Type = 'github' Owner = 'NixOS' Repo = 'nixpkgs' + +# Used to enable searching for programs in multi-program packages +[Importer.Sources.nixpkgs.Programs] +# Enable searching for programs in multi-program packages +Enable = true +# Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file +Attribute = 'programs.sqlite' diff --git a/go.mod b/go.mod index 8cd0dc5..b4e535f 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( go.alanpearce.eu/x v0.0.0-20241203124832-a29434dba11a go.uber.org/zap v1.27.0 golang.org/x/net v0.33.0 + modernc.org/sqlite v1.34.4 ) require ( @@ -43,13 +44,19 @@ require ( github.com/blevesearch/zapx/v14 v14.3.10 // indirect github.com/blevesearch/zapx/v15 v15.3.17 // indirect github.com/blevesearch/zapx/v16 v16.1.10 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/golang/geo v0.0.0-20230421003525-6adc56603217 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mschoch/smat v0.2.0 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/sykesm/zap-logfmt v0.0.4 // indirect github.com/thessem/zap-prettyconsole v0.5.2 // indirect go.etcd.io/bbolt v1.3.11 // indirect @@ -57,5 +64,11 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.36.0 // indirect + modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/strutil v1.2.0 // indirect + modernc.org/token v1.1.0 // indirect moul.io/zapfilter v1.7.0 // indirect ) diff --git a/go.sum b/go.sum index d1c3212..e5bc8ba 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/crewjam/csp v0.0.2/go.mod h1:0tirp4wHwMLZZtV+HXRqGFkUO7uD2ux+1ECvK+7/ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/evanw/esbuild v0.14.11/go.mod h1:GG+zjdi59yh3ehDn4ZWfPcATxjPDUH53iU4ZJbp7dkY= github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= @@ -72,13 +74,21 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -88,6 +98,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/osdevisnot/sorvor v0.4.4 h1:hcMWsWOKpUtDUE3F7dra1Jf12ftLHfgDcxlyPeVlz0Y= github.com/osdevisnot/sorvor v0.4.4/go.mod h1:D/j+vvJEmjIXndJf37uwFWD0Hjcq9DiGojyt4yMo7H0= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= @@ -100,6 +112,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= @@ -155,6 +169,8 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -176,6 +192,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -191,6 +208,8 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -206,5 +225,31 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.34.4 h1:sjdARozcL5KJBvYQvLlZEmctRgW9xqIZc2ncN7PU0P8= +modernc.org/sqlite v1.34.4/go.mod h1:3QQFCG2SEMtc2nv+Wq4cQCH7Hjcg+p/RMlS1XK+zwbk= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= moul.io/zapfilter v1.7.0 h1:7aFrG4N72bDH9a2BtYUuUaDS981Dxu3qybWfeqaeBDU= moul.io/zapfilter v1.7.0/go.mod h1:M+N2s+qZiA+bzRoyKMVRxyuERijS2ovi2pnMyiOGMvc= diff --git a/gomod2nix.toml b/gomod2nix.toml index df88513..5d3f8e4 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -79,6 +79,9 @@ schema = 3 [mod."github.com/crewjam/csp"] version = "v0.0.2" hash = "sha256-4vlGmDdQjPiXmueCV51fJH/hRcG8eqhCi9TENCXjzfA=" + [mod."github.com/dustin/go-humanize"] + version = "v1.0.1" + hash = "sha256-yuvxYYngpfVkUg9yAmG99IUVmADTQA0tMbBXe0Fq0Mc=" [mod."github.com/fsnotify/fsnotify"] version = "v1.8.0" hash = "sha256-+Rxg5q17VaqSU1xKPgurq90+Z1vzXwMLIBSe5UsyI/M=" @@ -94,9 +97,18 @@ schema = 3 [mod."github.com/golang/snappy"] version = "v0.0.4" hash = "sha256-Umx+5xHAQCN/Gi4HbtMhnDCSPFAXSsjVbXd8n5LhjAA=" + [mod."github.com/google/uuid"] + version = "v1.6.0" + hash = "sha256-VWl9sqUzdOuhW0KzQlv0gwwUQClYkmZwSydHG2sALYw=" + [mod."github.com/hashicorp/golang-lru/v2"] + version = "v2.0.7" + hash = "sha256-t1bcXLgrQNOYUVyYEZ0knxcXpsTk4IuJZDjKvyJX75g=" [mod."github.com/json-iterator/go"] version = "v1.1.12" hash = "sha256-To8A0h+lbfZ/6zM+2PpRpY3+L6725OPC66lffq6fUoM=" + [mod."github.com/mattn/go-isatty"] + version = "v0.0.20" + hash = "sha256-qhw9hWtU5wnyFyuMbKx+7RB8ckQaFQ8D+8GKPkN3HHQ=" [mod."github.com/mitchellh/mapstructure"] version = "v1.5.0" hash = "sha256-ztVhGQXs67MF8UadVvG72G3ly0ypQW0IRDdOOkjYwoE=" @@ -109,6 +121,9 @@ schema = 3 [mod."github.com/mschoch/smat"] version = "v0.2.0" hash = "sha256-DZvUJXjIcta3U+zxzgU3wpoGn/V4lpBY7Xme8aQUi+E=" + [mod."github.com/ncruces/go-strftime"] + version = "v0.1.9" + hash = "sha256-T0iw+UEckzueWHT88PkTnZZixyKCEa+DTLzIiiohuWY=" [mod."github.com/osdevisnot/sorvor"] version = "v0.4.4" hash = "sha256-BhyO7bvwxIdEV+c6Eo1uqahhcgsHiS8nJpg2aT8t+8s=" @@ -118,6 +133,9 @@ schema = 3 [mod."github.com/pkg/errors"] version = "v0.9.1" hash = "sha256-mNfQtcrQmu3sNg/7IwiieKWOgFQOVVe2yXgKBpe/wZw=" + [mod."github.com/remyoudompheng/bigfft"] + version = "v0.0.0-20230129092748-24d4a6f8daec" + hash = "sha256-vYmpyCE37eBYP/navhaLV4oX4/nu0Z/StAocLIFqrmM=" [mod."github.com/stoewer/go-strcase"] version = "v1.3.0" hash = "sha256-X0ilcefeqVQ44B9WT6euCMcigs7oLFypOQaGI33kGr8=" @@ -154,6 +172,27 @@ schema = 3 [mod."google.golang.org/protobuf"] version = "v1.36.0" hash = "sha256-7LfDxB2/x6sSzJdQ3sixWMaY0WZ/juwuz3rPBJxNzXQ=" + [mod."modernc.org/gc/v3"] + version = "v3.0.0-20240107210532-573471604cb6" + hash = "sha256-UO6/mPf3y3Iz5wnDAIRWHiCRQ+ElTECVbXXaekUkxA8=" + [mod."modernc.org/libc"] + version = "v1.55.3" + hash = "sha256-MGEOCkVDhjZW0t68m5p45UjikILl59KoL/3wx65O1zs=" + [mod."modernc.org/mathutil"] + version = "v1.6.0" + hash = "sha256-lfuEiS1odd2TWrTylnaGihSJ9myqKs3FLdpvd7PqTnE=" + [mod."modernc.org/memory"] + version = "v1.8.0" + hash = "sha256-ucvPr73zg8LjvU+bcoIPKTgwgcon3U9VhKrLEMH81xg=" + [mod."modernc.org/sqlite"] + version = "v1.34.4" + hash = "sha256-YuBVm9IN5bDm5LqH6kG0qTnHBMk213KF4+kyJUo5KDA=" + [mod."modernc.org/strutil"] + version = "v1.2.0" + hash = "sha256-NTYIWMRZjHmR77LMvsFOMCitt7toKTfH+zChYAMzZ2Y=" + [mod."modernc.org/token"] + version = "v1.1.0" + hash = "sha256-m8WyXJ9Mdw6B43wmy2+3HE7zHEi9ocBrhwe/eq+zdu8=" [mod."moul.io/zapfilter"] version = "v1.7.0" hash = "sha256-H6j5h8w123Y7d0zvKGkL5jiRYICtjmgzd2P/eeNaLrs=" diff --git a/internal/components/packageDetail.templ b/internal/components/packageDetail.templ index 65c74aa..84d2bdf 100644 --- a/internal/components/packageDetail.templ +++ b/internal/components/packageDetail.templ @@ -1,8 +1,6 @@ package components -import ( - "go.alanpearce.eu/searchix/internal/nix" -) +import "go.alanpearce.eu/searchix/internal/nix" func licenseName(l nix.License) string { if l.FullName != "" { @@ -32,6 +30,18 @@ templ PackageDetail(pkg nix.Package) { <code>{ pkg.MainProgram }</code> </dd> } + if len(pkg.Programs) > 0 { + <dt>Programs</dt> + <dd> + <ul> + for _, p := range pkg.Programs { + <li> + <code>{ p }</code> + </li> + } + </ul> + </dd> + } if len(pkg.Homepages) > 0 { <dt>Homepage</dt> <dd> diff --git a/internal/config/default.go b/internal/config/default.go index a857799..5202678 100644 --- a/internal/config/default.go +++ b/internal/config/default.go @@ -113,8 +113,12 @@ var DefaultConfig = Config{ Fetcher: ChannelNixpkgs, Channel: "nixos-unstable", OutputPath: "packages.json.br", - Timeout: Duration{5 * time.Minute}, + Timeout: Duration{15 * time.Minute}, Repo: nixpkgs, + Programs: ProgramsDB{ + Enable: true, + Attribute: "programs.sqlite", + }, }, }, }, diff --git a/internal/config/structs.go b/internal/config/structs.go index b31e0cd..e73425b 100644 --- a/internal/config/structs.go +++ b/internal/config/structs.go @@ -48,6 +48,12 @@ type Source struct { Timeout Duration `comment:"Abort import if it takes longer than this."` OutputPath string `comment:"(Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json."` Repo Repository `comment:"Used to generate declaration/definition links"` + Programs ProgramsDB `comment:"Used to enable searching for programs in multi-program packages"` +} + +type ProgramsDB struct { + Enable bool `comment:"Enable searching for programs in multi-program packages"` + Attribute string `comment:"Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file"` } func (source *Source) String() string { diff --git a/internal/importer/main.go b/internal/importer/main.go index 4c66501..dfa2477 100644 --- a/internal/importer/main.go +++ b/internal/importer/main.go @@ -11,6 +11,7 @@ import ( "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/fetcher" "go.alanpearce.eu/searchix/internal/index" + "go.alanpearce.eu/searchix/internal/programs" "go.alanpearce.eu/x/log" "github.com/pkg/errors" @@ -78,6 +79,17 @@ func createSourceImporter( ) if sourceMeta.Updated.After(previousUpdate) || forceUpdate { + var pdb *programs.DB + + if source.Programs.Enable { + pdb, err = programs.Instantiate(ctx, source, log.Named("programs")) + if err != nil { + logger.Warn("programs database instantiation failed", "error", err) + } + if pdb.Path != sourceMeta.ProgramsPath { + sourceMeta.ProgramsPath = pdb.Path + } + } err = setRepoRevision(files.Revision, source) if err != nil { @@ -105,6 +117,7 @@ func createSourceImporter( files.Packages, source, logger.Named("processor"), + pdb, ) } if err != nil { diff --git a/internal/importer/package.go b/internal/importer/package.go index 80adc38..59bccd8 100644 --- a/internal/importer/package.go +++ b/internal/importer/package.go @@ -9,6 +9,7 @@ import ( "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/nix" + "go.alanpearce.eu/searchix/internal/programs" "go.alanpearce.eu/x/log" "github.com/bcicen/jstream" @@ -39,12 +40,13 @@ type maintainerJSON struct { } type PackageIngester struct { - dec *jstream.Decoder - ms *mapstructure.Decoder - log *log.Logger - pkg packageJSON - infile io.ReadCloser - source *config.Source + dec *jstream.Decoder + ms *mapstructure.Decoder + log *log.Logger + pkg packageJSON + infile io.ReadCloser + source *config.Source + programs *programs.DB } func makeAdhocLicense(name string) nix.License { @@ -66,13 +68,15 @@ func NewPackageProcessor( infile io.ReadCloser, source *config.Source, log *log.Logger, + programsDB *programs.DB, ) (*PackageIngester, error) { i := &PackageIngester{ - dec: jstream.NewDecoder(infile, 2).EmitKV(), - log: log, - pkg: packageJSON{}, - infile: infile, - source: source, + dec: jstream.NewDecoder(infile, 2).EmitKV(), + log: log, + pkg: packageJSON{}, + infile: infile, + source: source, + programs: programsDB, } ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ @@ -116,6 +120,14 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, < results := make(chan nix.Importable) errs := make(chan error) + if i.programs != nil { + err := i.programs.Open() + if err != nil { + errs <- errors.WithMessage(err, "could not open programs database") + i.programs = nil + } + } + go func() { defer i.infile.Close() defer close(results) @@ -124,6 +136,7 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, < outer: for mv := range i.dec.Stream() { var err error + var programs []string select { case <-ctx.Done(): break outer @@ -216,6 +229,13 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, < continue } + if i.programs != nil { + programs, err = i.programs.GetPackagePrograms(ctx, kv.Key) + if err != nil { + errs <- errors.WithMessagef(err, "failed to get programs for package %s", i.pkg.Name) + } + } + maintainers := make([]nix.Maintainer, len(i.pkg.Meta.Maintainers)) for i, m := range i.pkg.Meta.Maintainers { maintainers[i] = nix.Maintainer{ @@ -250,8 +270,13 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, < Licenses: licenses, Maintainers: maintainers, Definition: url, + Programs: programs, } } + + if i.programs != nil { + i.programs.Close() + } }() return results, errs diff --git a/internal/index/index_meta.go b/internal/index/index_meta.go index e67c6f2..7d133cd 100644 --- a/internal/index/index_meta.go +++ b/internal/index/index_meta.go @@ -11,12 +11,13 @@ import ( "github.com/pkg/errors" ) -const CurrentSchemaVersion = 2 +const CurrentSchemaVersion = 3 type SourceMeta struct { - Updated time.Time - Path string - Rev string + Updated time.Time + Path string + Rev string + ProgramsPath string } type data struct { diff --git a/internal/index/indexer.go b/internal/index/indexer.go index 0c12104..9c291b8 100644 --- a/internal/index/indexer.go +++ b/internal/index/indexer.go @@ -126,6 +126,7 @@ func createIndexMapping() (mapping.IndexMapping, error) { packageMapping.AddFieldMappingsAt("MainProgram", keywordFieldMapping) packageMapping.AddFieldMappingsAt("PackageSet", keywordFieldMapping) packageMapping.AddFieldMappingsAt("Platforms", keywordFieldMapping) + packageMapping.AddFieldMappingsAt("Programs", keywordFieldMapping) indexMapping.AddDocumentMapping("option", optionMapping) indexMapping.AddDocumentMapping("package", packageMapping) diff --git a/internal/index/search.go b/internal/index/search.go index 9d77488..a62f484 100644 --- a/internal/index/search.go +++ b/internal/index/search.go @@ -164,6 +164,11 @@ func (index *ReadIndex) Search( query.AddShould(q) } + programsQuery := bleve.NewMatchQuery(keyword) + programsQuery.SetField("Programs") + programsQuery.Analyzer = "keyword_single" + query.AddShould(programsQuery) + attrQuery := bleve.NewMatchQuery(keyword) attrQuery.SetField("Attribute") attrQuery.Analyzer = "keyword_single" diff --git a/internal/nix/package.go b/internal/nix/package.go index 42bf77a..0b6519e 100644 --- a/internal/nix/package.go +++ b/internal/nix/package.go @@ -11,6 +11,7 @@ type Package struct { Licenses []License LongDescription Markdown MainProgram string + Programs []string Maintainers []Maintainer PackageSet string Platforms []string diff --git a/internal/programs/programs.go b/internal/programs/programs.go new file mode 100644 index 0000000..1dbfff7 --- /dev/null +++ b/internal/programs/programs.go @@ -0,0 +1,97 @@ +package programs + +import ( + "context" + "database/sql" + "fmt" + "os/exec" + "strings" + + "github.com/pkg/errors" + "go.alanpearce.eu/searchix/internal/config" + "go.alanpearce.eu/x/log" + _ "modernc.org/sqlite" //nolint:blank-imports // sqlite driver needed for database/sql +) + +type DB struct { + Path string + Source *config.Source + + logger *log.Logger + db *sql.DB +} + +func Instantiate(ctx context.Context, source *config.Source, logger *log.Logger) (*DB, error) { + // nix-instantiate --eval --json -I nixpkgs=channel:nixos-unstable --expr 'toString <nixpkgs/programs.sqlite>' + args := []string{ + "--eval", + "--json", + "-I", fmt.Sprintf("%s=channel:%s", source.Key, source.Channel), + "--expr", fmt.Sprintf("toString <%s/%s>", source.Key, source.Programs.Attribute), + } + + logger.Debug("nix-instantiate command", "args", args) + cmd := exec.CommandContext(ctx, "nix-instantiate", args...) + out, err := cmd.Output() + if err != nil { + return nil, errors.WithMessage(err, "failed to run nix-instantiate") + } + + outPath := strings.Trim(strings.TrimSpace(string(out)), "\"") + logger.Debug("got output path", "outputPath", outPath) + + return &DB{ + Source: source, + Path: outPath, + + logger: logger, + }, nil +} + +func (p *DB) Open() error { + db, err := sql.Open("sqlite", p.Path) + if err != nil { + return errors.WithMessage(err, "failed to open sqlite database") + } + p.db = db + + return nil +} + +func (p *DB) Close() error { + if err := p.db.Close(); err != nil { + return errors.WithMessage(err, "failed to close sqlite database") + } + + return nil +} + +func (p *DB) GetPackagePrograms(ctx context.Context, pkg string) (programs []string, err error) { + if p.db == nil { + return nil, errors.New("database not open") + } + rows, err := p.db.QueryContext(ctx, ` +SELECT name +FROM Programs +WHERE package = ? +GROUP BY name, package`, pkg) + if err != nil { + return nil, errors.WithMessage(err, "failed to execute query") + } + defer rows.Close() + + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, errors.WithMessage(err, "failed to scan row") + } + + programs = append(programs, name) + } + rerr := rows.Close() + if rerr != nil { + return nil, errors.WithMessage(rerr, "sql error") + } + + return +} |