all repos — searchix @ 9c9aa6b408812248c9f8c29ecc3546ef64947ea8

Search engine for NixOS, nix-darwin, home-manager and NUR users

perf: iterate over results without keeping the set in memory

Alan Pearce
commit

9c9aa6b408812248c9f8c29ecc3546ef64947ea8

parent

e8fbdf3bd12c8920a6e9bd84b34e787764b11eaf

M go.modgo.mod
@@ -17,7 +17,7 @@ github.com/pelletier/go-toml/v2 v2.2.3
github.com/stoewer/go-strcase v1.3.0 github.com/yuin/goldmark v1.7.8 gitlab.com/tozd/go/errors v0.10.0 - go.alanpearce.eu/gomponents v1.3.0 + go.alanpearce.eu/gomponents v1.4.0 go.alanpearce.eu/x v0.0.0-20241203124832-a29434dba11a go.uber.org/zap v1.27.0 golang.org/x/net v0.33.0
M go.sumgo.sum
@@ -140,8 +140,8 @@ github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic=
github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= gitlab.com/tozd/go/errors v0.10.0 h1:A98kL+gaDvWnY6ZB/u8zP+sYaWsWUGBHeFMtamvW/74= gitlab.com/tozd/go/errors v0.10.0/go.mod h1:q3Ugr0C8dCzMEkrzjjlV2qNsm9e0KvqBjwcbcjCpBe4= -go.alanpearce.eu/gomponents v1.3.0 h1:yfrDWy77KF/ohDLmgNNud1hMR9WWOvCZJlfyO4SSDo4= -go.alanpearce.eu/gomponents v1.3.0/go.mod h1:uX96UAsHCut1cKMAYVWWxQ9ADt1CAPI8LpyAu0LRQPs= +go.alanpearce.eu/gomponents v1.4.0 h1:Ibvoce+U0rnPKlDOE+wXDbNniNQL8mYO667+qS5J1Go= +go.alanpearce.eu/gomponents v1.4.0/go.mod h1:WxD+6FRSvwThQOzV0r6zPAA9CRb41lutUZMSC7r6BRc= go.alanpearce.eu/x v0.0.0-20241203124832-a29434dba11a h1:NUv3AzGxwMVSq26takww8/nyl+sPO2BsESoVSU8G49U= go.alanpearce.eu/x v0.0.0-20241203124832-a29434dba11a/go.mod h1:FRM6J9HMQ/RV2Q5j+6RKBYWh/YNeEUriGSqDRchiHuQ= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
M gomod2nix.tomlgomod2nix.toml
@@ -152,8 +152,8 @@ [mod."gitlab.com/tozd/go/errors"]
version = "v0.10.0" hash = "sha256-oW37KsieVKJOWk9ZXbGuQvuU4nyJCZzgYrTZHFkoCs4=" [mod."go.alanpearce.eu/gomponents"] - version = "v1.3.0" - hash = "sha256-rZz5rJdm58axukN6RlaVKSJ9v2TPngIHt3P1APpXSxY=" + version = "v1.4.0" + hash = "sha256-Q8YN8eNouMnW/JaBJpjUmaOZ5cfBj5gFURgVlbnaiDM=" [mod."go.alanpearce.eu/x"] version = "v0.0.0-20241203124832-a29434dba11a" hash = "sha256-ojqWkz3VqeAOevFxOTO5S3acRItCA4pUrTaul887+x8="
M internal/components/combined.gointernal/components/combined.go
@@ -36,7 +36,7 @@ ),
), ), TBody( - g.Map(result.Hits, func(hit index.DocumentMatch) g.Node { + g.MapIter(result.Hits, func(hit index.DocumentMatch) g.Node { return Tr( Td( openCombinedDialogLink(nix.GetKey(hit.Data)),
M internal/components/options.gointernal/components/options.go
@@ -21,7 +21,7 @@ ),
), ), TBody( - g.Map(result.Hits, func(hit index.DocumentMatch) g.Node { + g.MapIter(result.Hits, func(hit index.DocumentMatch) g.Node { if m := convertMatch[nix.Option](hit.Data); m != nil { return optionRow(hit, *m) }
M internal/components/packages.gointernal/components/packages.go
@@ -22,7 +22,7 @@ ),
), ), TBody( - g.Map(result.Hits, func(hit index.DocumentMatch) g.Node { + g.MapIter(result.Hits, func(hit index.DocumentMatch) g.Node { if m := convertMatch[nix.Package](hit.Data); m != nil { return packageRow(hit, *m) }
M internal/index/search.gointernal/index/search.go
@@ -4,6 +4,7 @@ import (
"bytes" "context" "encoding/gob" + "iter" "time" "go.alanpearce.eu/searchix/internal/config"
@@ -25,7 +26,7 @@ }
type Result struct { *bleve.SearchResult - Hits []DocumentMatch + Hits iter.Seq[DocumentMatch] } type ReadIndex struct {
@@ -87,24 +88,31 @@ if err != nil {
return nil, errors.WithMessage(err, "failed to execute search query") } - results := make([]DocumentMatch, bleveResult.Hits.Len()) - var buf bytes.Buffer - for i, result := range bleveResult.Hits { - results[i].DocumentMatch = bleveResult.Hits[i] - _, err = buf.WriteString(result.Fields["_data"].(string)) - if err != nil { - return nil, errors.WithMessage(err, "error fetching result data") + hits := func(yield func(DocumentMatch) bool) { + var buf bytes.Buffer + for _, match := range bleveResult.Hits { + hit := DocumentMatch{ + DocumentMatch: match, + Data: nil, + } + _, err := buf.WriteString(match.Fields["_data"].(string)) + if err != nil { + index.log.Warn("error fetching result data", "error", err) + } + err = gob.NewDecoder(&buf).Decode(&hit.Data) + if err != nil { + index.log.Warn("error decoding gob data", "error", err, "data", buf.String()) + } + buf.Reset() + if !yield(hit) { + return + } } - err = gob.NewDecoder(&buf).Decode(&results[i].Data) - if err != nil { - return nil, errors.WithMessagef(err, "error decoding gob data: %s", buf.String()) - } - buf.Reset() } return &Result{ SearchResult: bleveResult, - Hits: results, + Hits: hits, }, nil } }
@@ -188,6 +196,7 @@ ) (*nix.Importable, errors.E) {
key := nix.MakeKey(source, id) query := bleve.NewDocIDQuery([]string{key}) search := bleve.NewSearchRequest(query) + search.Size = 1 result, err := index.search(ctx, search) if err != nil {
@@ -198,5 +207,11 @@ if result.Total == 0 {
return nil, nil } - return &result.Hits[0].Data, err + for hit := range result.Hits { + if hit.ID == key { + return &hit.Data, err + } + } + + return nil, err }