From c15b142b18dcdc7f5ab6d5f1afca8ae1696692cc Mon Sep 17 00:00:00 2001 From: Alan Pearce Date: Tue, 7 May 2024 17:33:06 +0200 Subject: feat: search one set of options --- internal/search/search.go | 123 ++++++++++++++++++++++++++++++++++++++++++++++ internal/server/server.go | 43 +++++++++------- 2 files changed, 149 insertions(+), 17 deletions(-) create mode 100644 internal/search/search.go (limited to 'internal') diff --git a/internal/search/search.go b/internal/search/search.go new file mode 100644 index 0000000..fc53ad0 --- /dev/null +++ b/internal/search/search.go @@ -0,0 +1,123 @@ +package search + +import ( + "context" + "log" + "os" + "path" + "sync" + + "searchix/internal/options" + + "github.com/bcicen/jstream" + "github.com/blevesearch/bleve/v2" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +const maxResults = 10 + +var index bleve.Index +var docs sync.Map + +type Result[T options.NixOption] struct { + *bleve.SearchResult + Results []T +} + +func Index() error { + bleve.SetLog(log.Default()) + textFieldMapping := bleve.NewTextFieldMapping() + nameMapping := bleve.NewTextFieldMapping() + // something special for tokenisation? + // nameMapping. + optionMapping := bleve.NewDocumentStaticMapping() + + optionMapping.AddFieldMappingsAt("Option", nameMapping) + optionMapping.AddFieldMappingsAt("RelatedPackages", textFieldMapping) + optionMapping.AddFieldMappingsAt("Description", textFieldMapping) + + indexMapping := bleve.NewIndexMapping() + indexMapping.AddDocumentMapping("option", optionMapping) + + var err error + index, err = bleve.NewMemOnly(indexMapping) + // index, err = bleve.New(path.Join(cfg.DataPath, const indexFilename = "index.bleve"), indexMapping) + + if err != nil { + return errors.WithMessage(err, "error opening index") + } + batch := index.NewBatch() + _ = batch + + jsonFile, err := os.Open(path.Join("data", "processed", "darwin-options.json")) + if err != nil { + return errors.WithMessage(err, "error opening json file") + } + + dec := jstream.NewDecoder(jsonFile, 1) + var opt options.NixOption + ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + ErrorUnused: true, + ZeroFields: true, + Result: &opt, + }) + if err != nil { + return errors.WithMessage(err, "could not create struct decoder") + } + for mv := range dec.Stream() { + err := ms.Decode(mv.Value) // stores in opt + + if err != nil { + return errors.WithMessagef(err, "could not decode object into option, object: %#v", mv.Value) + } + + docs.Store(opt.Option, opt) + + err = batch.Index(opt.Option, opt) + if err != nil { + return errors.WithMessagef(err, "could not index option %s", opt.Option) + } + } + err = index.Batch(batch) + if err != nil { + return errors.WithMessage(err, "failed to run batch index operation") + } + + return nil +} + +func Search[T options.NixOption](ctx context.Context, keyword string) (*Result[T], error) { + query := bleve.NewMatchQuery(keyword) + search := bleve.NewSearchRequest(query) + + bleveResult, err := index.SearchInContext(ctx, search) + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + if err != nil { + return nil, errors.WithMessage(err, "failed to execute search query") + } + + results := make([]T, min(maxResults, bleveResult.Total)) + for i, result := range bleveResult.Hits { + doc, _ := docs.Load(result.ID) + results[i] = doc.(T) + if i > maxResults { + break + } + } + + return &Result[T]{ + bleveResult, + results, + }, nil + } +} + +func Load(name string) any { + doc, _ := docs.Load(name) + + return doc +} diff --git a/internal/server/server.go b/internal/server/server.go index a6f22e2..3f87b9d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -2,7 +2,6 @@ package server import ( "context" - "encoding/json" "fmt" "html/template" "io" @@ -17,6 +16,7 @@ import ( cfg "searchix/internal/config" "searchix/internal/options" + "searchix/internal/search" "github.com/getsentry/sentry-go" sentryhttp "github.com/getsentry/sentry-go/http" @@ -59,10 +59,10 @@ type TemplateData struct { Query string } -type OptionResultData struct { +type ResultData[T options.NixOption] struct { TemplateData Query string - Results options.NixOptions + Results *search.Result[T] } func applyDevModeOverrides(config *cfg.Config) { @@ -70,6 +70,13 @@ func applyDevModeOverrides(config *cfg.Config) { config.CSP.ConnectSrc = slices.Insert(config.CSP.ConnectSrc, 0, "'self'") } +func init() { + err := search.Index() + if err != nil { + log.Fatalf("could not build search index, error: %#v", err) + } +} + func New(runtimeConfig *Config) (*Server, error) { var err error config, err = cfg.GetConfig() @@ -113,23 +120,25 @@ func New(runtimeConfig *Config) (*Server, error) { } }) - var nixosOptions = options.NixOptions{} - jsonFile, err := os.ReadFile(path.Join(config.DataPath, "test.json")) - if err != nil { - slog.Error(fmt.Sprintf("error reading json file: %v", err)) - } - err = json.Unmarshal(jsonFile, &nixosOptions) - if err != nil { - slog.Error(fmt.Sprintf("error parsing json file: %v", err)) - } - + timeout := 1 * time.Second mux.HandleFunc("/options/results", func(w http.ResponseWriter, r *http.Request) { - tdata := OptionResultData{ + ctx, cancel := context.WithTimeoutCause(r.Context(), timeout, errors.New("timeout")) + defer cancel() + results, err := search.Search(ctx, r.URL.Query().Get("query")) + if err != nil { + if err == context.DeadlineExceeded { + http.Error(w, "Search timed out", http.StatusInternalServerError) + + return + } + slog.Error("search error", "error", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + } + tdata := ResultData[options.NixOption]{ TemplateData: indexData, Query: r.URL.Query().Get("query"), - Results: nixosOptions, + Results: results, } - var err error if r.Header.Get("Fetch") == "true" { w.Header().Add("Content-Type", "text/html; charset=utf-8") err = templates["options"].Execute(w, tdata) @@ -137,7 +146,7 @@ func New(runtimeConfig *Config) (*Server, error) { err = templates["options"].ExecuteTemplate(w, "index.gotmpl", tdata) } if err != nil { - slog.Error(fmt.Sprintf("template error: %v", err)) + slog.Error("template error", "template", "options", "error", err) http.Error(w, err.Error(), http.StatusInternalServerError) } }) -- cgit 1.4.1