From e062ca72b222b890e345548bd8422d5df98e9fef Mon Sep 17 00:00:00 2001 From: Alan Pearce Date: Thu, 9 May 2024 16:47:41 +0200 Subject: feat: import sources from configuration in go code and index options --- config.toml | 38 ++++++ data/processed/.keep | 0 data/raw/.keep | 0 default.nix | 4 - go.mod | 1 + go.sum | 3 + gomod2nix.toml | 3 + import/main.go | 111 ++++++++++++++++ importers/nixos-options.nix | 45 ------- internal/config/config.go | 22 ++++ internal/importer/channel.go | 82 ++++++++++++ internal/importer/http.go | 63 ++++++++++ internal/importer/importer.go | 112 +++++++++++++++++ internal/importer/ingest.go | 237 +++++++++++++++++++++++++++++++++++ internal/importer/nixpkgs-channel.go | 82 ++++++++++++ internal/importer/repository.go | 44 +++++++ internal/importer/source-type.go | 44 +++++++ internal/options/option.go | 5 +- internal/options/process.go | 224 --------------------------------- internal/search/indexer.go | 183 +++++++++++++++++++++++++++ internal/search/search.go | 150 ++++++---------------- internal/server/server.go | 70 +++++------ justfile | 40 +----- nix/overlays/bleve.nix | 39 ++++++ nix/overlays/default.nix | 1 + process/main.go | 49 -------- shell.nix | 1 + 27 files changed, 1142 insertions(+), 511 deletions(-) create mode 100644 config.toml delete mode 100644 data/processed/.keep delete mode 100644 data/raw/.keep create mode 100644 import/main.go delete mode 100644 importers/nixos-options.nix create mode 100644 internal/importer/channel.go create mode 100644 internal/importer/http.go create mode 100644 internal/importer/importer.go create mode 100644 internal/importer/ingest.go create mode 100644 internal/importer/nixpkgs-channel.go create mode 100644 internal/importer/repository.go create mode 100644 internal/importer/source-type.go delete mode 100644 internal/options/process.go create mode 100644 internal/search/indexer.go create mode 100644 nix/overlays/bleve.nix delete mode 100644 process/main.go diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..b865415 --- /dev/null +++ b/config.toml @@ -0,0 +1,38 @@ +[sources.nixos] + name = "NixOS" + enable = true + type = "channel" + channel = "nixos-unstable" + import-path = "nixos/release.nix" + attribute = "options" + output-path = "share/doc/nixos/options.json" + [sources.repo] + type = "github" + owner = "NixOS" + repo = "nixpkgs" + +[sources.darwin] + name = "darwin" + enable = true + type = "channel" + channel = "darwin" + import-path = "release.nix" + attribute = "options" + output-path = "share/doc/darwin/options.json" + [sources.repo] + type = "github" + owner = "LnL7" + repo = "nix-darwin" + +[sources.home-manager] + name = "home-manager" + enable = true + type = "channel" + channel = "home-manager" + import-path = "default.nix" + attribute = "docs.json" + output-path = "share/doc/home-manager/options.json" + [sources.repo] + type = "github" + owner = "nix-community" + repo = "home-manager" diff --git a/data/processed/.keep b/data/processed/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/data/raw/.keep b/data/raw/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/default.nix b/default.nix index 38bbf9e..10f7ac7 100644 --- a/default.nix +++ b/default.nix @@ -15,10 +15,6 @@ in inherit (sources.simple-css) url sha256; }; - nixos-options = import (./. + "/importers/nixos-options.nix") { }; - darwin-options = (import { }).options; - home-manager-options = (import { }).docs.json; - searchix = pkgs.buildGoApplication { pname = "searchix"; version = "0.1"; diff --git a/go.mod b/go.mod index f8fa654..1a81e20 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/pelletier/go-toml/v2 v2.2.1 github.com/pkg/errors v0.9.1 github.com/shengyanli1982/law v0.1.15 + github.com/stoewer/go-strcase v1.3.0 github.com/yuin/goldmark v1.7.1 ) diff --git a/go.sum b/go.sum index 9f5792f..a61a78e 100644 --- a/go.sum +++ b/go.sum @@ -88,6 +88,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/shengyanli1982/law v0.1.15 h1:puSn0Saa+0ptjszspycfWHPSu0D3kBcU3oEeW83MRIc= github.com/shengyanli1982/law v0.1.15/go.mod h1:20k9YnOTwilUB4X5Z4S7TIX5Ek1Ok4xfx8V8ZxIWlyM= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -96,6 +98,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= diff --git a/gomod2nix.toml b/gomod2nix.toml index 6378cab..047d995 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -115,6 +115,9 @@ schema = 3 [mod."github.com/shengyanli1982/law"] version = "v0.1.15" hash = "sha256-Z5G3PtR7V0d04MN+kBge33Pv6VDjJryx+N7JGJkzfLQ=" + [mod."github.com/stoewer/go-strcase"] + version = "v1.3.0" + hash = "sha256-X0ilcefeqVQ44B9WT6euCMcigs7oLFypOQaGI33kGr8=" [mod."github.com/yuin/goldmark"] version = "v1.7.1" hash = "sha256-3EUgwoZRRs2jNBWSbB0DGNmfBvx7CeAgEwyUdaRaeR4=" diff --git a/import/main.go b/import/main.go new file mode 100644 index 0000000..de652c5 --- /dev/null +++ b/import/main.go @@ -0,0 +1,111 @@ +package main + +import ( + "context" + "errors" + "log" + "log/slog" + "os" + "os/exec" + "path" + "searchix/internal/config" + "searchix/internal/importer" + "searchix/internal/search" + "slices" + "strings" + "time" +) + +const timeout = 30 * time.Minute + +func main() { + if _, found := os.LookupEnv("DEBUG"); found { + slog.SetLogLoggerLevel(slog.LevelDebug) + } + cfg, err := config.GetConfig() + if err != nil { + log.Fatal(err) + } + + enabledSources := slices.DeleteFunc(cfg.Sources, func(s importer.Source) bool { + return !s.Enable + }) + + if len(enabledSources) == 0 { + slog.Info("No sources enabled") + + return + } + + indexer, err := search.NewIndexer(cfg.DataPath) + if err != nil { + log.Fatalf("Failed to create indexer: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + var imp importer.Importer + var hadErrors bool + for _, source := range enabledSources { + logger := slog.With("name", source.Name, "importer", source.Type.String()) + logger.Debug("starting importer") + + switch source.Type { + case importer.ChannelNixpkgs: + imp = importer.NewNixpkgsChannelImporter(source, cfg.DataPath, logger) + case importer.Channel: + imp = importer.NewChannelImporter(source, cfg.DataPath, logger) + default: + log.Printf("unsupported importer type %s", source.Type.String()) + + continue + } + + updated, err := imp.FetchIfNeeded(ctx) + + if err != nil { + var exerr *exec.ExitError + if errors.As(err, &exerr) { + lines := strings.Split(strings.TrimSpace(string(exerr.Stderr)), "\n") + for _, line := range lines { + logger.Warn("importer fetch failed", "stderr", line, "status", exerr.ExitCode()) + } + } else { + logger.Warn("importer fetch failed", "error", err) + } + hadErrors = true + + continue + } + logger.Info("importer fetch succeeded", "updated", updated) + + if updated { + hadWarnings, err := imp.Import(ctx, indexer) + + if err != nil { + msg := err.Error() + for _, line := range strings.Split(strings.TrimSpace(msg), "\n") { + logger.Error("importer init failed", "error", line) + } + + continue + } + if hadWarnings { + logger.Warn("importer succeeded, but with warnings/errors") + } else { + logger.Info("importer succeeded") + } + } + } + + err = indexer.Close() + if err != nil { + slog.Error("error closing indexer", "error", err) + } + + if hadErrors { + os.RemoveAll(path.Join(cfg.DataPath, "index.bleve")) + defer os.Exit(1) + } +} diff --git a/importers/nixos-options.nix b/importers/nixos-options.nix deleted file mode 100644 index 3c0a18e..0000000 --- a/importers/nixos-options.nix +++ /dev/null @@ -1,45 +0,0 @@ -{ nixpkgs ? -, pkgs ? import nixpkgs { } -, system ? builtins.currentSystem -, stateVersion ? pkgs.lib.version -, ... -}: -let - inherit (pkgs) lib; - inherit (lib) hasPrefix removePrefix; - - nixos = pkgs.nixos ({ lib, ... }: { - nixpkgs.hostPlatform = system; - system.stateVersion = lib.versions.majorMinor stateVersion; - }); - - inherit (nixos.config.system.nixos) revision; - - gitHubDeclaration = user: repo: ref: subpath: - # Default to `master` if we don't know what revision the system - # configuration is using (custom nixpkgs, etc.). - let urlRef = if ref != null then ref else "master"; - in { - url = "https://github.com/${user}/${repo}/blob/${urlRef}/${subpath}"; - name = "<${repo}/${subpath}>"; - }; - - doc = pkgs.nixosOptionsDoc { - inherit (nixos) options; - transformOptions = opt: opt // { - declarations = - map - (decl: - if hasPrefix (toString nixpkgs) (toString decl) - then - gitHubDeclaration "NixOS" "nixpkgs" revision - (removePrefix "/" - (removePrefix (toString nixpkgs) (toString decl))) - else if decl == "lib/modules.nix" then - gitHubDeclaration "NixOS" "nixpkgs" revision decl - else decl) - opt.declarations; - }; - }; -in -doc.optionsJSON diff --git a/internal/config/config.go b/internal/config/config.go index 5b06efa..2717291 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,9 +2,11 @@ package config import ( "log/slog" + "maps" "net/url" "os" "searchix/internal/file" + "searchix/internal/importer" "github.com/pelletier/go-toml/v2" "github.com/pkg/errors" @@ -28,6 +30,7 @@ type Config struct { DataPath string `toml:"data_path"` CSP CSP `toml:"content-security-policy"` Headers map[string]string + Sources map[string]importer.Source } var defaultConfig = Config{ @@ -38,6 +41,22 @@ var defaultConfig = Config{ Headers: map[string]string{ "x-content-type-options": "nosniff", }, + Sources: map[string]importer.Source{ + "nixos": importer.Source{ + Name: "NixOS", + Enable: true, + Type: importer.Channel, + Channel: "nixos-unstable", + ImportPath: "nixos/release.nix", + Attribute: "options", + OutputPath: "share/doc/nixos/options.json", + Repo: importer.Repository{ + Type: "github", + Owner: "NixOS", + Repo: "nixpkgs", + }, + }, + }, } func GetConfig() (*Config, error) { @@ -60,6 +79,9 @@ func GetConfig() (*Config, error) { return nil, errors.Wrap(err, "config error") } } + maps.DeleteFunc(config.Sources, func(_ string, v importer.Source) bool { + return !v.Enable + }) return &config, nil } diff --git a/internal/importer/channel.go b/internal/importer/channel.go new file mode 100644 index 0000000..4d051cc --- /dev/null +++ b/internal/importer/channel.go @@ -0,0 +1,82 @@ +package importer + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/exec" + "path" + "searchix/internal/file" + "searchix/internal/search" + "strconv" + "strings" + + "github.com/pkg/errors" +) + +type ChannelImporter struct { + DataPath string + Source Source + SourceFile string + Logger *slog.Logger + indexPath string +} + +func (i *ChannelImporter) FetchIfNeeded(parent context.Context) (bool, error) { + ctx, cancel := context.WithTimeout(parent, i.Source.FetchTimeout) + defer cancel() + + dest := i.DataPath + + before, err := os.Readlink(dest) + if file.NeedNotExist(err) != nil { + return false, errors.WithMessagef(err, "could not call readlink on file %s", dest) + } + i.Logger.Debug("stat before", "name", before) + + args := []string{ + "--no-build-output", + "--timeout", + strconv.Itoa(int(i.Source.FetchTimeout.Seconds() - 1)), + fmt.Sprintf("<%s/%s>", i.Source.Channel, i.Source.ImportPath), + "--attr", + i.Source.Attribute, + "--out-link", + dest, + } + + i.Logger.Debug("nix-build command", "args", args) + cmd := exec.CommandContext(ctx, "nix-build", args...) + out, err := cmd.Output() + if err != nil { + return false, errors.WithMessage(err, "failed to run nix-build (--dry-run)") + } + i.Logger.Debug("nix-build", "output", strings.TrimSpace(string(out))) + + outPath := path.Join(dest, i.Source.OutputPath) + i.Logger.Debug("checking output path", "outputPath", outPath, "dest", dest, "source", i.Source.OutputPath) + after, err := os.Readlink(dest) + if err := file.NeedNotExist(err); err != nil { + return false, errors.WithMessagef(err, "failed to stat output file from nix-build, filename: %s", outPath) + } + i.Logger.Debug("stat after", "name", after) + + return before != after, nil +} + +func (i *ChannelImporter) Import(parent context.Context, indexer *search.WriteIndex) (bool, error) { + if i.Source.OutputPath == "" { + return false, errors.New("no output path specified") + } + + filename := path.Join(i.DataPath, i.SourceFile, i.Source.OutputPath) + i.Logger.Debug("preparing import run", "revision", i.Source.Repo.Revision, "filename", filename) + + return processOptions(parent, indexer, &importConfig{ + IndexPath: i.indexPath, + Source: i.Source, + Filename: filename, + Logger: i.Logger, + }) +} diff --git a/internal/importer/http.go b/internal/importer/http.go new file mode 100644 index 0000000..1bf2428 --- /dev/null +++ b/internal/importer/http.go @@ -0,0 +1,63 @@ +package importer + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "os" + "searchix/internal/file" + "strings" + "time" + + "github.com/pkg/errors" +) + +func fetchFileIfNeeded(ctx context.Context, path string, url string) (needed bool, err error) { + stat, err := file.StatIfExists(path) + if err != nil { + return false, errors.WithMessagef(err, "could not stat file %s", path) + } + + var mtime string + if stat != nil { + mtime = strings.Replace(stat.ModTime().UTC().Format(time.RFC1123), "UTC", "GMT", 1) + } + + req, err := http.NewRequestWithContext(ctx, "GET", url, http.NoBody) + if err != nil { + return false, errors.WithMessagef(err, "could not create HTTP request for %s", url) + } + + if mtime != "" { + req.Header.Set("If-Modified-Since", mtime) + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return false, errors.WithMessagef(err, "could not make HTTP request to %s", url) + } + defer res.Body.Close() + + switch res.StatusCode { + case http.StatusNotModified: + needed = false + case http.StatusOK: + newMtime, err := time.Parse(time.RFC1123, res.Header.Get("Last-Modified")) + if err != nil { + slog.Warn("could not parse Last-Modified header from response", "value", res.Header.Get("Last-Modified")) + } + err = file.WriteToFile(path, res.Body) + if err != nil { + return false, errors.WithMessagef(err, "could not write response body to file %s", path) + } + err = os.Chtimes(path, time.Time{}, newMtime) + if err != nil { + slog.Warn("could not update mtime on file", "file", path) + } + needed = true + default: + return false, fmt.Errorf("got response code %d, don't know what to do", res.StatusCode) + } + + return needed, nil +} diff --git a/internal/importer/importer.go b/internal/importer/importer.go new file mode 100644 index 0000000..2318fe4 --- /dev/null +++ b/internal/importer/importer.go @@ -0,0 +1,112 @@ +package importer + +import ( + "context" + "log/slog" + "path" + "searchix/internal/search" + "sync" + "time" +) + +type Source struct { + Name string + Enable bool + Type Type + Channel string + Attribute string + ImportPath string `toml:"import-path"` + FetchTimeout time.Duration `toml:"fetch-timeout"` + ImportTimeout time.Duration `toml:"import-timeout"` + OutputPath string `toml:"output-path"` + Repo Repository +} + +type Importer interface { + FetchIfNeeded(context.Context) (bool, error) + Import(context.Context, *search.WriteIndex) (bool, error) +} + +func NewNixpkgsChannelImporter(source Source, dataPath string, logger *slog.Logger) *NixpkgsChannelImporter { + indexPath := dataPath + fullpath := path.Join(dataPath, source.Channel) + + return &NixpkgsChannelImporter{ + DataPath: fullpath, + Source: source, + Logger: logger, + indexPath: indexPath, + } +} + +func NewChannelImporter(source Source, dataPath string, logger *slog.Logger) *ChannelImporter { + indexPath := dataPath + fullpath := path.Join(dataPath, source.Channel) + + return &ChannelImporter{ + DataPath: fullpath, + Source: source, + Logger: logger, + indexPath: indexPath, + } +} + +type importConfig struct { + IndexPath string + Filename string + Source Source + Logger *slog.Logger +} + +func processOptions(parent context.Context, indexer *search.WriteIndex, conf *importConfig) (bool, error) { + ctx, cancel := context.WithTimeout(parent, conf.Source.ImportTimeout) + defer cancel() + + conf.Logger.Debug("creating option processor", "filename", conf.Filename) + processor, err := NewOptionProcessor(conf.Filename, conf.Source) + if err != nil { + return true, err + } + + wg := sync.WaitGroup{} + + wg.Add(1) + options, pErrs := processor.Process(ctx) + + wg.Add(1) + iErrs := indexer.ImportOptions(ctx, options) + + var hadErrors bool + go func() { + for { + select { + case err, running := <-iErrs: + if !running { + wg.Done() + iErrs = nil + slog.Info("ingest completed") + + continue + } + hadErrors = true + conf.Logger.Warn("error ingesting option", "error", err) + case err, running := <-pErrs: + if !running { + wg.Done() + pErrs = nil + slog.Info("processing completed") + + continue + } + hadErrors = true + conf.Logger.Warn("error processing option", "error", err) + } + } + }() + + slog.Debug("options processing", "state", "waiting") + wg.Wait() + slog.Debug("options processing", "state", "complete") + + return hadErrors, nil +} diff --git a/internal/importer/ingest.go b/internal/importer/ingest.go new file mode 100644 index 0000000..b9db80c --- /dev/null +++ b/internal/importer/ingest.go @@ -0,0 +1,237 @@ +package importer + +import ( + "context" + "fmt" + "log/slog" + "net/url" + "os" + "reflect" + "searchix/internal/options" + "strings" + + "github.com/bcicen/jstream" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +type nixValueJSON struct { + Type string `mapstructure:"_type"` + Text string +} + +type linkJSON struct { + Name string + URL string `json:"url"` +} + +type nixOptionJSON struct { + Declarations []linkJSON + Default *nixValueJSON + Description string + Example *nixValueJSON + Loc []string + ReadOnly bool + RelatedPackages string + Type string +} + +func ValueTypeToString(valueType jstream.ValueType) string { + switch valueType { + case jstream.Unknown: + return "unknown" + case jstream.Null: + return "null" + case jstream.String: + return "string" + case jstream.Number: + return "number" + case jstream.Boolean: + return "boolean" + case jstream.Array: + return "array" + case jstream.Object: + return "object" + } + + return "very strange" +} + +func makeGitHubFileURL(userRepo string, ref string, subPath string) string { + url, _ := url.JoinPath("https://github.com/", userRepo, "blob", ref, subPath) + + return url +} + +// make configurable? +var channelRepoMap = map[string]string{ + "nixpkgs": "NixOS/nixpkgs", + "nix-darwin": "LnL7/nix-darwin", + "home-manager": "nix-community/home-manager", +} + +func MakeChannelLink(channel string, ref string, subPath string) (*options.Link, error) { + if channelRepoMap[channel] == "" { + return nil, fmt.Errorf("don't know what repository relates to channel <%s>", channel) + } + + return &options.Link{ + Name: fmt.Sprintf("<%s/%s>", channel, subPath), + URL: makeGitHubFileURL(channelRepoMap[channel], ref, subPath), + }, nil +} + +func convertNixValue(nj *nixValueJSON) *options.NixValue { + if nj == nil { + return nil + } + switch nj.Type { + case "", "literalExpression": + return &options.NixValue{ + Text: nj.Text, + } + case "literalMD": + return &options.NixValue{ + Markdown: options.Markdown(nj.Text), + } + default: + slog.Warn("got unexpected NixValue type", "type", nj.Type, "text", nj.Text) + + return nil + } +} + +type OptionIngester struct { + dec *jstream.Decoder + ms *mapstructure.Decoder + optJSON nixOptionJSON + infile *os.File + source Source +} + +type Ingester[T options.NixOption] interface { + Process() (<-chan *T, <-chan error) +} + +func NewOptionProcessor(inpath string, source Source) (*OptionIngester, error) { + infile, err := os.Open(inpath) + if err != nil { + return nil, errors.WithMessagef(err, "failed to open input file %s", inpath) + } + i := OptionIngester{ + dec: jstream.NewDecoder(infile, 1).EmitKV(), + optJSON: nixOptionJSON{}, + infile: infile, + source: source, + } + + ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + ErrorUnused: true, + ZeroFields: true, + Result: &i.optJSON, + Squash: true, + DecodeHook: mapstructure.TextUnmarshallerHookFunc(), + }) + if err != nil { + defer infile.Close() + + return nil, errors.WithMessage(err, "could not create mapstructure decoder") + } + i.ms = ms + + return &i, nil +} + +func (i *OptionIngester) Process(ctx context.Context) (<-chan *options.NixOption, <-chan error) { + results := make(chan *options.NixOption) + errs := make(chan error) + + go func() { + defer i.infile.Close() + defer close(results) + defer close(errs) + + slog.Debug("starting decoder stream") + outer: + for mv := range i.dec.Stream() { + select { + case <-ctx.Done(): + break outer + default: + } + if err := i.dec.Err(); err != nil { + errs <- errors.WithMessage(err, "could not decode JSON") + + continue + } + if mv.ValueType != jstream.Object { + errs <- errors.Errorf("unexpected object type %s", ValueTypeToString(mv.ValueType)) + + continue + } + kv := mv.Value.(jstream.KV) + x := kv.Value.(map[string]interface{}) + + var decls []*options.Link + for _, decl := range x["declarations"].([]interface{}) { + i.optJSON = nixOptionJSON{} + + switch decl := reflect.ValueOf(decl); decl.Kind() { + case reflect.String: + s := decl.String() + link, err := MakeChannelLink(i.source.Channel, i.source.Repo.Revision, s) + if err != nil { + errs <- errors.WithMessagef(err, + "could not make a channel link for channel %s, revision %s and subpath %s", + i.source.Channel, i.source.Repo.Revision, s, + ) + + continue + } + decls = append(decls, link) + case reflect.Map: + v := decl.Interface().(map[string]interface{}) + link := options.Link{ + Name: v["name"].(string), + URL: v["url"].(string), + } + decls = append(decls, &link) + default: + errs <- errors.Errorf("unexpected declaration type %s", decl.Kind().String()) + + continue + } + } + if len(decls) > 0 { + x["declarations"] = decls + } + + err := i.ms.Decode(x) // stores in optJSON + if err != nil { + errs <- errors.WithMessagef(err, "failed to decode option %#v", x) + + continue + } + + var decs = make([]options.Link, len(i.optJSON.Declarations)) + for i, d := range i.optJSON.Declarations { + decs[i] = options.Link(d) + } + + // slog.Debug("sending option", "name", kv.Key) + results <- &options.NixOption{ + Option: kv.Key, + Source: strings.ToLower(i.source.Name), + Declarations: decs, + Default: convertNixValue(i.optJSON.Default), + Description: options.Markdown(i.optJSON.Description), + Example: convertNixValue(i.optJSON.Example), + RelatedPackages: options.Markdown(i.optJSON.RelatedPackages), + Loc: i.optJSON.Loc, + Type: i.optJSON.Type, + } + } + }() + + return results, errs +} diff --git a/internal/importer/nixpkgs-channel.go b/internal/importer/nixpkgs-channel.go new file mode 100644 index 0000000..0e5be62 --- /dev/null +++ b/internal/importer/nixpkgs-channel.go @@ -0,0 +1,82 @@ +package importer + +import ( + "bytes" + "context" + "log/slog" + "net/url" + "os" + "path" + "searchix/internal/file" + "searchix/internal/search" + + "github.com/pkg/errors" +) + +type NixpkgsChannelImporter struct { + DataPath string + Source Source + Logger *slog.Logger + indexPath string +} + +func makeChannelURL(channel string, subPath string) (string, error) { + url, err := url.JoinPath("https://channels.nixos.org/", channel, subPath) + + return url, errors.WithMessagef(err, "error creating URL") +} + +var filesToFetch = map[string]string{ + "revision": "git-revision", + "options": "options.json.br", +} + +func (i *NixpkgsChannelImporter) FetchIfNeeded(parent context.Context) (bool, error) { + ctx, cancel := context.WithTimeout(parent, i.Source.FetchTimeout) + defer cancel() + + root := i.DataPath + + err := file.Mkdirp(root) + if err != nil { + return false, errors.WithMessagef(err, "error creating directory for data: %s", root) + } + + for _, filename := range filesToFetch { + url, err := makeChannelURL(i.Source.Channel, filename) + if err != nil { + return false, err + } + + path := path.Join(root, filename) + + updated, err := fetchFileIfNeeded(ctx, path, url) + if err != nil { + return false, err + } + // don't bother to issue requests for the later files + if !updated { + return false, err + } + } + + return true, nil +} + +func (i *NixpkgsChannelImporter) Import(parent context.Context, indexer *search.WriteIndex) (bool, error) { + filename := path.Join(i.DataPath, filesToFetch["options"]) + revFilename := path.Join(i.DataPath, filesToFetch["revision"]) + bits, err := os.ReadFile(revFilename) + if err != nil { + return false, errors.WithMessagef(err, "unable to read revision file at %s", revFilename) + } + i.Source.Repo.Revision = string(bytes.TrimSpace(bits)) + i.Logger.Debug("preparing import run", "revision", i.Source.Repo.Revision, "filename", filename) + + return processOptions(parent, indexer, &importConfig{ + IndexPath: i.indexPath, + Source: i.Source, + Filename: filename, + Logger: i.Logger, + }) +} diff --git a/internal/importer/repository.go b/internal/importer/repository.go new file mode 100644 index 0000000..6cfd55e --- /dev/null +++ b/internal/importer/repository.go @@ -0,0 +1,44 @@ +package importer + +import ( + "fmt" + "strings" +) + +type RepoType int + +const ( + GitHub = iota + 1 +) + +type Repository struct { + Type string `default:"github"` + Owner string + Repo string + Revision string +} + +func (f RepoType) String() string { + switch f { + case GitHub: + return "github" + default: + return fmt.Sprintf("RepoType(%d)", f) + } +} + +func parseRepoType(name string) (RepoType, error) { + switch strings.ToLower(name) { + case "github": + return GitHub, nil + default: + return Unknown, fmt.Errorf("unsupported repo type %s", name) + } +} + +func (f *RepoType) UnmarshalText(text []byte) error { + var err error + *f, err = parseRepoType(string(text)) + + return err +} diff --git a/internal/importer/source-type.go b/internal/importer/source-type.go new file mode 100644 index 0000000..5d84547 --- /dev/null +++ b/internal/importer/source-type.go @@ -0,0 +1,44 @@ +package importer + +import ( + "fmt" + + "github.com/stoewer/go-strcase" +) + +type Type int + +const ( + Unknown = iota + Channel + ChannelNixpkgs +) + +func (f Type) String() string { + switch f { + case Channel: + return "channel" + case ChannelNixpkgs: + return "channel-nixpkgs" + } + + return fmt.Sprintf("Fetcher(%d)", f) +} + +func parseType(name string) (Type, error) { + switch strcase.KebabCase(name) { + case "channel": + return Channel, nil + case "channel-nixpkgs": + return ChannelNixpkgs, nil + default: + return Unknown, fmt.Errorf("unsupported fetcher %s", name) + } +} + +func (f *Type) UnmarshalText(text []byte) error { + var err error + *f, err = parseType(string(text)) + + return err +} diff --git a/internal/options/option.go b/internal/options/option.go index a43dd49..b7fe818 100644 --- a/internal/options/option.go +++ b/internal/options/option.go @@ -14,6 +14,7 @@ type Link struct { type NixOption struct { Option string + Source string Declarations []Link Default *NixValue `json:",omitempty"` @@ -24,4 +25,6 @@ type NixOption struct { Type string } -type NixOptions []NixOption +func (*NixOption) BleveType() string { + return "option" +} diff --git a/internal/options/process.go b/internal/options/process.go deleted file mode 100644 index 4e7c664..0000000 --- a/internal/options/process.go +++ /dev/null @@ -1,224 +0,0 @@ -package options - -import ( - "encoding/json" - "fmt" - "io" - "log/slog" - "net/url" - "os" - "reflect" - - "github.com/bcicen/jstream" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" -) - -type nixValueJSON struct { - Type string `mapstructure:"_type"` - Text string -} - -type linkJSON struct { - Name string - URL string `json:"url"` -} - -type nixOptionJSON struct { - Declarations []linkJSON - Default *nixValueJSON - Description string - Example *nixValueJSON - Loc []string - ReadOnly bool - RelatedPackages string - Type string -} - -func ValueTypeToString(valueType jstream.ValueType) string { - switch valueType { - case jstream.Unknown: - return "unknown" - case jstream.Null: - return "null" - case jstream.String: - return "string" - case jstream.Number: - return "number" - case jstream.Boolean: - return "boolean" - case jstream.Array: - return "array" - case jstream.Object: - return "object" - } - - return "very strange" -} - -func makeGitHubFileURL(userRepo string, ref string, subPath string) string { - url, _ := url.JoinPath("https://github.com/", userRepo, "blob", ref, subPath) - - return url -} - -// make configurable? -var channelRepoMap = map[string]string{ - "nixpkgs": "NixOS/nixpkgs", - "nix-darwin": "LnL7/nix-darwin", - "home-manager": "nix-community/home-manager", -} - -func MakeChannelLink(channel string, ref string, subPath string) (*Link, error) { - if channelRepoMap[channel] == "" { - return nil, fmt.Errorf("don't know what repository relates to channel <%s>", channel) - } - - return &Link{ - Name: fmt.Sprintf("<%s/%s>", channel, subPath), - URL: makeGitHubFileURL(channelRepoMap[channel], ref, subPath), - }, nil -} - -func convertNixValue(nj *nixValueJSON) *NixValue { - if nj == nil { - return nil - } - switch nj.Type { - case "", "literalExpression": - return &NixValue{ - Text: nj.Text, - } - case "literalMD": - return &NixValue{ - Markdown: Markdown(nj.Text), - } - default: - slog.Warn("got unexpected NixValue type", "type", nj.Type, "text", nj.Text) - - return nil - } -} - -func Process(inpath string, outpath string, channel string, revision string) error { - infile, err := os.Open(inpath) - if err != nil { - return errors.WithMessagef(err, "failed to open input file %s", inpath) - } - defer infile.Close() - outfile, err := os.Create(outpath) - if err != nil { - return errors.WithMessagef(err, "failed to open output file %s", outpath) - } - if outpath != "/dev/stdout" { - defer outfile.Close() - } - - dec := jstream.NewDecoder(infile, 1).EmitKV() - var optJSON nixOptionJSON - ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - ErrorUnused: true, - ZeroFields: true, - Result: &optJSON, - Squash: true, - DecodeHook: mapstructure.TextUnmarshallerHookFunc(), - }) - if err != nil { - return errors.WithMessage(err, "could not create mapstructure decoder") - } - - _, err = outfile.WriteString("[\n") - if err != nil { - return errors.WithMessage(err, "could not write to output") - } - for mv := range dec.Stream() { - if err := dec.Err(); err != nil { - return errors.WithMessage(err, "could not decode JSON") - } - if mv.ValueType != jstream.Object { - return errors.Errorf("unexpected object type %s", ValueTypeToString(mv.ValueType)) - } - kv := mv.Value.(jstream.KV) - x := kv.Value.(map[string]interface{}) - - var decls []*Link - for _, decl := range x["declarations"].([]interface{}) { - optJSON = nixOptionJSON{} - - switch decl := reflect.ValueOf(decl); decl.Kind() { - case reflect.String: - s := decl.String() - link, err := MakeChannelLink(channel, revision, s) - if err != nil { - return errors.WithMessagef(err, - "could not make a channel link for channel %s, revision %s and subpath %s", - channel, revision, s, - ) - } - decls = append(decls, link) - case reflect.Map: - v := decl.Interface().(map[string]interface{}) - link := Link{ - Name: v["name"].(string), - URL: v["url"].(string), - } - decls = append(decls, &link) - default: - println("kind", decl.Kind().String()) - panic("unexpected object type") - } - } - if len(decls) > 0 { - x["declarations"] = decls - } - - err = ms.Decode(x) // stores in optJSON - if err != nil { - return errors.WithMessagef(err, "failed to decode option %#v", x) - } - - var decs = make([]Link, len(optJSON.Declarations)) - for i, d := range optJSON.Declarations { - decs[i] = Link(d) - } - - opt := NixOption{ - Option: kv.Key, - Declarations: decs, - Default: convertNixValue(optJSON.Default), - Description: Markdown(optJSON.Description), - Example: convertNixValue(optJSON.Example), - RelatedPackages: Markdown(optJSON.RelatedPackages), - Loc: optJSON.Loc, - Type: optJSON.Type, - } - - b, err := json.MarshalIndent(opt, "", " ") - if err != nil { - return errors.WithMessagef(err, "failed to encode option %#v", opt) - } - - _, err = outfile.Write(b) - if err != nil { - return errors.WithMessage(err, "failed to write to output") - } - _, err = outfile.WriteString(",\n") - if err != nil { - return errors.WithMessage(err, "failed to write to output") - } - } - - if outpath != "/dev/stdout" { - _, err = outfile.Seek(-2, io.SeekCurrent) - if err != nil { - return errors.WithMessage(err, "could not write to output") - } - } - - _, err = outfile.WriteString("\n]\n") - if err != nil { - return errors.WithMessage(err, "could not write to output") - } - - return nil -} diff --git a/internal/search/indexer.go b/internal/search/indexer.go new file mode 100644 index 0000000..b0e57d4 --- /dev/null +++ b/internal/search/indexer.go @@ -0,0 +1,183 @@ +package search + +import ( + "bytes" + "context" + "encoding/gob" + "log" + "log/slog" + "path" + "searchix/internal/options" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/analysis/analyzer/web" + "github.com/blevesearch/bleve/v2/analysis/token/camelcase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/letter" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/single" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/mapping" + index "github.com/blevesearch/bleve_index_api" + "github.com/pkg/errors" +) + +type WriteIndex struct { + index bleve.Index + indexMapping *mapping.IndexMappingImpl +} + +func NewIndexer(dir string) (*WriteIndex, error) { + var err error + bleve.SetLog(log.Default()) + + indexPath := path.Join(dir, indexFilename) + + indexMapping := bleve.NewIndexMapping() + indexMapping.StoreDynamic = false + indexMapping.IndexDynamic = false + indexMapping.TypeField = "BleveType" + + textFieldMapping := bleve.NewTextFieldMapping() + textFieldMapping.Store = false + + descriptionFieldMapping := bleve.NewTextFieldMapping() + descriptionFieldMapping.Store = false + descriptionFieldMapping.Analyzer = web.Name + + err = indexMapping.AddCustomAnalyzer("option_name", map[string]interface{}{ + "type": custom.Name, + "tokenizer": letter.Name, + "token_filters": []string{ + camelcase.Name, + }, + }) + if err != nil { + return nil, errors.WithMessage(err, "could not add custom analyser") + } + err = indexMapping.AddCustomAnalyzer("loc", map[string]interface{}{ + "type": keyword.Name, + "tokenizer": letter.Name, + "token_filters": []string{ + camelcase.Name, + }, + }) + if err != nil { + return nil, errors.WithMessage(err, "could not add custom analyser") + } + err = indexMapping.AddCustomAnalyzer("keyword_single", map[string]interface{}{ + "type": keyword.Name, + "tokenizer": single.Name, + }) + if err != nil { + return nil, errors.WithMessage(err, "could not add custom analyser") + } + + keywordFieldMapping := bleve.NewKeywordFieldMapping() + keywordFieldMapping.Analyzer = "keyword_single" + + nameMapping := bleve.NewTextFieldMapping() + nameMapping.Analyzer = "option_name" + nameMapping.IncludeTermVectors = true + nameMapping.Store = false + + nixValueMapping := bleve.NewDocumentStaticMapping() + nixValueMapping.AddFieldMappingsAt("Text", textFieldMapping) + nixValueMapping.AddFieldMappingsAt("Markdown", textFieldMapping) + + locFieldMapping := bleve.NewKeywordFieldMapping() + locFieldMapping.Analyzer = "loc" + locFieldMapping.IncludeTermVectors = true + locFieldMapping.Store = false + + optionMapping := bleve.NewDocumentStaticMapping() + + optionMapping.AddFieldMappingsAt("Option", keywordFieldMapping) + optionMapping.AddFieldMappingsAt("Source", keywordFieldMapping) + optionMapping.AddFieldMappingsAt("Loc", locFieldMapping) + optionMapping.AddFieldMappingsAt("RelatedPackages", textFieldMapping) + optionMapping.AddFieldMappingsAt("Description", textFieldMapping) + + optionMapping.AddSubDocumentMapping("Default", nixValueMapping) + optionMapping.AddSubDocumentMapping("Example", nixValueMapping) + + indexMapping.AddDocumentMapping("option", optionMapping) + + idx, err := bleve.New(indexPath, indexMapping) + if err != nil { + return nil, errors.WithMessagef(err, "unable to create index at path %s", indexPath) + } + + return &WriteIndex{ + idx, + indexMapping, + }, nil +} + +func (i *WriteIndex) ImportOptions(ctx context.Context, objects <-chan *options.NixOption) <-chan error { + var err error + errs := make(chan error) + + go func() { + defer close(errs) + batch := i.index.NewBatch() + + outer: + for opt := range objects { + select { + case <-ctx.Done(): + slog.Debug("context cancelled") + + break outer + default: + } + + doc := document.NewDocument(opt.Source + "/" + opt.Option) + err = i.indexMapping.MapDocument(doc, opt) + if err != nil { + errs <- errors.WithMessagef(err, "could not map document for option: %s", opt.Option) + + continue + } + + var data bytes.Buffer + enc := gob.NewEncoder(&data) + err = enc.Encode(opt) + if err != nil { + errs <- errors.WithMessage(err, "could not store option in search index") + + continue + } + field := document.NewTextFieldWithIndexingOptions("_data", nil, data.Bytes(), index.StoreField) + newDoc := doc.AddField(field) + + // slog.Debug("adding option to index", "name", opt.Option) + err = batch.IndexAdvanced(newDoc) + + if err != nil { + errs <- errors.WithMessagef(err, "could not index option %s", opt.Option) + + continue + } + } + + size := batch.Size() + slog.Debug("flushing batch", "size", size) + + err := i.index.Batch(batch) + if err != nil { + errs <- errors.WithMessagef(err, "could not flush batch") + } + }() + + return errs +} + +func (i *WriteIndex) Close() error { + err := i.index.Close() + if err != nil { + return errors.WithMessagef(err, "could not close index") + } + + return nil +} diff --git a/internal/search/search.go b/internal/search/search.go index 97d8404..92afdfb 100644 --- a/internal/search/search.go +++ b/internal/search/search.go @@ -4,151 +4,73 @@ import ( "bytes" "context" "encoding/gob" - "log" - "os" "path" "searchix/internal/options" - "github.com/bcicen/jstream" "github.com/blevesearch/bleve/v2" - "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" - "github.com/blevesearch/bleve/v2/analysis/token/camelcase" - "github.com/blevesearch/bleve/v2/analysis/tokenizer/letter" - "github.com/blevesearch/bleve/v2/document" "github.com/blevesearch/bleve/v2/search" - index "github.com/blevesearch/bleve_index_api" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) const ResultsPerPage = 20 +const indexFilename = "index.bleve" -type DocumentMatch[T options.NixOption] struct { +type DocumentMatch struct { search.DocumentMatch - Data T + Data options.NixOption } -type Result[T options.NixOption] struct { +type Result struct { *bleve.SearchResult - Hits []DocumentMatch[T] + Hits []DocumentMatch } -type Index[T options.NixOption] struct { +type ReadIndex struct { index bleve.Index } -func New[T options.NixOption](kind string) (*Index[T], error) { - var err error - bleve.SetLog(log.Default()) +func Open(dir string) (*ReadIndex, error) { + indexPath := path.Join(dir, indexFilename) - indexMapping := bleve.NewIndexMapping() - - textFieldMapping := bleve.NewTextFieldMapping() - textFieldMapping.Store = false - - descriptionFieldMapping := bleve.NewTextFieldMapping() - descriptionFieldMapping.Store = false - descriptionFieldMapping.Analyzer = "web" - - err = indexMapping.AddCustomAnalyzer("option_name", map[string]interface{}{ - "type": custom.Name, - "tokenizer": letter.Name, - "token_filters": []string{ - camelcase.Name, - }, - }) + idx, err := bleve.Open(indexPath) if err != nil { - return nil, errors.WithMessage(err, "could not add custom analyser") + return nil, errors.WithMessagef(err, "unable to open index at path %s", indexPath) } - nameMapping := bleve.NewTextFieldMapping() - nameMapping.Analyzer = "option_name" - nameMapping.IncludeTermVectors = true - nameMapping.Store = false - - nixValueMapping := bleve.NewDocumentStaticMapping() - nixValueMapping.AddFieldMappingsAt("Text", textFieldMapping) - nixValueMapping.AddFieldMappingsAt("Markdown", textFieldMapping) - - optionMapping := bleve.NewDocumentStaticMapping() - - optionMapping.AddFieldMappingsAt("Option", nameMapping) - optionMapping.AddFieldMappingsAt("Loc", bleve.NewKeywordFieldMapping()) - optionMapping.AddFieldMappingsAt("RelatedPackages", textFieldMapping) - optionMapping.AddFieldMappingsAt("Description", textFieldMapping) - - optionMapping.AddSubDocumentMapping("Default", nixValueMapping) - optionMapping.AddSubDocumentMapping("Example", nixValueMapping) - - indexMapping.AddDocumentMapping("option", optionMapping) - - idx, err := bleve.NewMemOnly(indexMapping) - // index, err = bleve.New(path.Join(cfg.DataPath, const indexFilename = "index.bleve"), indexMapping) - - if err != nil { - return nil, errors.WithMessage(err, "error opening index") - } - batch := idx.NewBatch() - - jsonFile, err := os.Open(path.Join("data", "processed", kind+".json")) - if err != nil { - return nil, errors.WithMessage(err, "error opening json file") - } - - dec := jstream.NewDecoder(jsonFile, 1) - var opt options.NixOption - ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - ErrorUnused: true, - ZeroFields: true, - Result: &opt, - }) - if err != nil { - return nil, errors.WithMessage(err, "could not create struct decoder") - } - for mv := range dec.Stream() { - opt = options.NixOption{} - orig := mv.Value.(map[string]interface{}) - err := ms.Decode(orig) // stores in opt - if err != nil { - return nil, errors.WithMessagef(err, "could not decode value: %+v", orig) - } - doc := document.NewDocument(opt.Option) - err = indexMapping.MapDocument(doc, opt) - if err != nil { - return nil, errors.WithMessagef(err, "could not map document for option: %s", opt.Option) - } + return &ReadIndex{ + idx, + }, nil +} - var data bytes.Buffer - enc := gob.NewEncoder(&data) - err = enc.Encode(opt) - if err != nil { - return nil, errors.WithMessage(err, "could not store option in search index") - } - field := document.NewTextFieldWithIndexingOptions("data", nil, data.Bytes(), index.StoreField) - newDoc := doc.AddField(field) +func (index *ReadIndex) GetSource(ctx context.Context, name string) (*bleve.SearchResult, error) { + query := bleve.NewTermQuery(name) + query.SetField("Source") + search := bleve.NewSearchRequest(query) - err = batch.IndexAdvanced(newDoc) + result, err := index.index.SearchInContext(ctx, search) + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: if err != nil { - return nil, errors.WithMessagef(err, "could not index option %s", opt.Option) + return nil, errors.WithMessagef(err, "failed to execute search to find source %s in index", name) } } - err = idx.Batch(batch) - if err != nil { - return nil, errors.WithMessage(err, "failed to run batch index operation") - } - return &Index[T]{ - idx, - }, nil + return result, nil } -func (index *Index[T]) Search(ctx context.Context, keyword string, from uint64) (*Result[T], error) { - query := bleve.NewMatchQuery(keyword) - query.Analyzer = "option_name" +func (index *ReadIndex) Search(ctx context.Context, source string, keyword string, from uint64) (*Result, error) { + sourceQuery := bleve.NewTermQuery(source) + userQuery := bleve.NewMatchQuery(keyword) + userQuery.Analyzer = "option_name" + + query := bleve.NewConjunctionQuery(sourceQuery, userQuery) + search := bleve.NewSearchRequest(query) search.Size = ResultsPerPage - search.Fields = []string{"data"} + search.Fields = []string{"_data"} search.Explain = true if from != 0 { @@ -164,10 +86,10 @@ func (index *Index[T]) Search(ctx context.Context, keyword string, from uint64) return nil, errors.WithMessage(err, "failed to execute search query") } - results := make([]DocumentMatch[T], min(ResultsPerPage, bleveResult.Total)) + results := make([]DocumentMatch, min(ResultsPerPage, bleveResult.Total)) var buf bytes.Buffer for i, result := range bleveResult.Hits { - _, err = buf.WriteString(result.Fields["data"].(string)) + _, err = buf.WriteString(result.Fields["_data"].(string)) if err != nil { return nil, errors.WithMessage(err, "error fetching result data") } @@ -178,7 +100,7 @@ func (index *Index[T]) Search(ctx context.Context, keyword string, from uint64) buf.Reset() } - return &Result[T]{ + return &Result{ SearchResult: bleveResult, Hits: results, }, nil diff --git a/internal/server/server.go b/internal/server/server.go index b794f05..5def347 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -20,6 +20,7 @@ import ( "searchix/internal/options" "searchix/internal/search" + "github.com/blevesearch/bleve/v2" "github.com/getsentry/sentry-go" sentryhttp "github.com/getsentry/sentry-go/http" "github.com/osdevisnot/sorvor/pkg/livereload" @@ -57,17 +58,18 @@ type Server struct { const jsSnippet = template.HTML(livereload.JsSnippet) // #nosec G203 type TemplateData struct { - LiveReload template.HTML - Source string - Query string - Results bool + LiveReload template.HTML + Source string + Query string + Results bool + SourceResult *bleve.SearchResult } type ResultData[T options.NixOption] struct { TemplateData Query string ResultsPerPage int - Results *search.Result[T] + Results *search.Result Prev string Next string } @@ -77,24 +79,6 @@ func applyDevModeOverrides(config *cfg.Config) { config.CSP.ConnectSrc = slices.Insert(config.CSP.ConnectSrc, 0, "'self'") } -var index = map[string]*search.Index[options.NixOption]{} - -var sourceFileName = map[string]string{ - "darwin": "darwin-options", - "home-manager": "home-manager-options", - "nixos": "nixos-options-nixos-unstable", -} - -func makeIndex(source string, filename string) { - var err error - slog.Info("loading index", "index", source) - index[source], err = search.New(filename) - slog.Info("loaded index", "index", source) - if err != nil { - log.Fatalf("could not build search index, error: %#v", err) - } -} - func New(runtimeConfig *Config) (*Server, error) { var err error config, err = cfg.GetConfig() @@ -102,6 +86,13 @@ func New(runtimeConfig *Config) (*Server, error) { return nil, errors.WithMessage(err, "error parsing configuration file") } + slog.Debug("loading index") + index, err := search.Open(config.DataPath) + slog.Debug("loaded index") + if err != nil { + log.Fatalf("could not open search index, error: %#v", err) + } + env := "development" if runtimeConfig.Production { env = "production" @@ -138,19 +129,34 @@ func New(runtimeConfig *Config) (*Server, error) { } }) + const getSourceTimeout = 1 * time.Second mux.HandleFunc("/options/{source}/search", func(w http.ResponseWriter, r *http.Request) { source := r.PathValue("source") - if index[source] == nil { + ctx, cancel := context.WithTimeout(context.Background(), getSourceTimeout) + defer cancel() + + sourceResult, err := index.GetSource(ctx, source) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + + return + } + + if sourceResult.Total == 0 { http.Error(w, "Unknown source", http.StatusNotFound) return } - err := templates["search"].Execute(w, TemplateData{ - LiveReload: jsSnippet, - Source: source, + + err = templates["search"].Execute(w, TemplateData{ + LiveReload: jsSnippet, + Source: source, + SourceResult: sourceResult, }) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) + + return } }) @@ -159,7 +165,7 @@ func New(runtimeConfig *Config) (*Server, error) { source := r.PathValue("source") ctx, cancel := context.WithTimeoutCause(r.Context(), timeout, errors.New("timeout")) defer cancel() - if index[source] == nil { + if index == nil { http.Error(w, "Unknown source", http.StatusNotFound) return @@ -173,7 +179,7 @@ func New(runtimeConfig *Config) (*Server, error) { http.Error(w, "Bad query string", http.StatusBadRequest) } } - results, err := index[source].Search(ctx, qs, (page-1)*search.ResultsPerPage) + results, err := index.Search(ctx, source, qs, (page-1)*search.ResultsPerPage) if err != nil { if err == context.DeadlineExceeded { http.Error(w, "Search timed out", http.StatusInternalServerError) @@ -238,12 +244,6 @@ func New(runtimeConfig *Config) (*Server, error) { mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("frontend/static")))) - go func() { - for source, filename := range sourceFileName { - makeIndex(source, filename) - } - }() - if runtimeConfig.LiveReload { applyDevModeOverrides(config) liveReload := livereload.New() diff --git a/justfile b/justfile index 700e255..78bea08 100644 --- a/justfile +++ b/justfile @@ -4,44 +4,6 @@ default: prepare: ln -sf $(nix-build --no-out-link -A css) frontend/static/base.css -get-nixpkgs-revision channel="nixos-unstable": - curl -L https://channels.nixos.org/{{ channel }}/git-revision > data/raw/nixpkgs-{{ channel }}-revision - -update-nixpkgs channel="nixos-unstable": (get-nixpkgs-revision channel) - curl -L https://channels.nixos.org/{{ channel }}/packages.json.br | brotli --stdout --decompress > data/raw/nixpkgs-{{ channel }}.json - -update-nixos-options channel="nixos-unstable": (get-nixpkgs-revision channel) - curl -L https://channels.nixos.org/{{ channel }}/options.json.br | brotli --stdout --decompress > data/raw/nixos-options-{{ channel }}.json - -update-darwin-options: - ln -sf $(nix-build --no-out-link -A darwin-options)/share/doc/darwin/options.json data/raw/darwin-options.json - -update-home-manager-options: - ln -sf $(nix-build --no-out-link -A home-manager-options)/share/doc/home-manager/options.json data/raw/home-manager-options.json - -process-nixos-options channel="nixos-unstable": - wgo run -exit ./process \ - --input data/raw/nixos-options-{{ channel }}.json \ - --output data/processed/nixos-options-{{ channel }}.json \ - --repo NixOS/nixpkgs \ - --revision-file data/raw/nixpkgs-{{ channel }}-revision - -process-darwin-options: - wgo run -exit ./process \ - --input data/raw/darwin-options.json \ - --output data/processed/darwin-options.json \ - --repo LnL7/nix-darwin - -process-home-manager-options: - wgo run -exit ./process \ - --input data/raw/home-manager-options.json \ - --output data/processed/home-manager-options.json \ - --repo nix-community/home-manager - -process-all-options: process-nixos-options process-darwin-options process-home-manager-options - -update-all-options: update-nixos-options update-darwin-options update-home-manager-options - checkformat: gofmt -d . goimports -d . @@ -56,5 +18,5 @@ fix: precommit: nix-build -A pre-commit-check -dev: prepare +dev: watchexec wgo run -exit ./serve/ --live diff --git a/nix/overlays/bleve.nix b/nix/overlays/bleve.nix new file mode 100644 index 0000000..8e36679 --- /dev/null +++ b/nix/overlays/bleve.nix @@ -0,0 +1,39 @@ +{ lib +, stdenv +, fetchFromGitHub +, installShellFiles +, buildGoModule +}: +let + gomod = builtins.fromTOML (builtins.readFile ./../../gomod2nix.toml); + version = gomod.mod."github.com/blevesearch/bleve/v2".version; +in +buildGoModule rec { + pname = "bleve"; + inherit version; + + src = fetchFromGitHub { + owner = "blevesearch"; + repo = "bleve"; + rev = version; + hash = "sha256-E7ykT0t4QTn615WfTE9EygD+p5kQQ3Qm7zZ/Jqb8tK8="; + }; + + vendorHash = "sha256-gkajiRCY+tPifBz5PRelFCZCfaWN/pti+7amuRmQI6Q="; + + subPackages = [ "cmd/bleve" ]; + + nativeBuildInputs = [ installShellFiles ]; + postInstall = lib.optionalString (stdenv.buildPlatform == stdenv.targetPlatform) '' + installShellCompletion --cmd bleve \ + --bash <($out/bin/bleve completion bash) \ + --fish <($out/bin/bleve completion fish) \ + --zsh <($out/bin/bleve completion zsh) + ''; + + meta = { + description = "Command-line tool to interact with bleve indexes"; + homepage = "http://blevesearch.com"; + licenses = lib.licenses.asl20; + }; +} diff --git a/nix/overlays/default.nix b/nix/overlays/default.nix index 3e5c925..db27c7e 100644 --- a/nix/overlays/default.nix +++ b/nix/overlays/default.nix @@ -3,4 +3,5 @@ self: super: { (final: prev: super.callPackage ./prettier-plugin-go-template { } ); + bleve = super.callPackage ./bleve.nix { }; } diff --git a/process/main.go b/process/main.go deleted file mode 100644 index 1089144..0000000 --- a/process/main.go +++ /dev/null @@ -1,49 +0,0 @@ -package main - -import ( - "log" - "log/slog" - "os" - "strings" - - "searchix/internal/options" - - "github.com/ardanlabs/conf/v3" - "github.com/pkg/errors" -) - -type Config struct { - Input string `conf:"short:i,required,help:NixOS options file (json)"` - Output string `conf:"short:o,default:/dev/stdout"` - Revision string `conf:"short:r,flag:revision,default:master"` - RevisionFile string `conf:"short:f,flag:revision-file"` - Channel string `conf:"short:c,flag:channel,default:nixpkgs"` -} - -func main() { - if os.Getenv("DEBUG") != "" { - slog.SetLogLoggerLevel(slog.LevelDebug) - } - log.SetFlags(0) - - config := Config{} - help, err := conf.Parse("", &config) - if err != nil { - if errors.Is(err, conf.ErrHelpWanted) { - log.Fatalln(help) - } - log.Fatalf("parsing command line: %v", err) - } - if config.RevisionFile != "" { - f, err := os.ReadFile(config.RevisionFile) - if err != nil { - log.Fatalf("Error reading revision file %s: %v", config.RevisionFile, err) - } - config.Revision = strings.TrimSpace(string(f)) - } - - err = options.Process(config.Input, config.Output, config.Channel, config.Revision) - if err != nil { - log.Fatalf("Error processing file: %v", err) - } -} diff --git a/shell.nix b/shell.nix index 97d8cda..2e85e8c 100644 --- a/shell.nix +++ b/shell.nix @@ -22,6 +22,7 @@ pkgs.mkShell { goEnv brotli + bleve wgo gomod2nix niv -- cgit 1.4.1