perf: iterate over results without keeping the set in memory
Alan Pearce alan@alanpearce.eu
Fri, 21 Mar 2025 21:55:00 +0100
7 files changed, 38 insertions(+), 23 deletions(-)
M go.mod → go.mod
@@ -17,7 +17,7 @@ github.com/pelletier/go-toml/v2 v2.2.3 github.com/stoewer/go-strcase v1.3.0 github.com/yuin/goldmark v1.7.8 gitlab.com/tozd/go/errors v0.10.0 - go.alanpearce.eu/gomponents v1.3.0 + go.alanpearce.eu/gomponents v1.4.0 go.alanpearce.eu/x v0.0.0-20241203124832-a29434dba11a go.uber.org/zap v1.27.0 golang.org/x/net v0.33.0
M go.sum → go.sum
@@ -140,8 +140,8 @@ github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic= github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= gitlab.com/tozd/go/errors v0.10.0 h1:A98kL+gaDvWnY6ZB/u8zP+sYaWsWUGBHeFMtamvW/74= gitlab.com/tozd/go/errors v0.10.0/go.mod h1:q3Ugr0C8dCzMEkrzjjlV2qNsm9e0KvqBjwcbcjCpBe4= -go.alanpearce.eu/gomponents v1.3.0 h1:yfrDWy77KF/ohDLmgNNud1hMR9WWOvCZJlfyO4SSDo4= -go.alanpearce.eu/gomponents v1.3.0/go.mod h1:uX96UAsHCut1cKMAYVWWxQ9ADt1CAPI8LpyAu0LRQPs= +go.alanpearce.eu/gomponents v1.4.0 h1:Ibvoce+U0rnPKlDOE+wXDbNniNQL8mYO667+qS5J1Go= +go.alanpearce.eu/gomponents v1.4.0/go.mod h1:WxD+6FRSvwThQOzV0r6zPAA9CRb41lutUZMSC7r6BRc= go.alanpearce.eu/x v0.0.0-20241203124832-a29434dba11a h1:NUv3AzGxwMVSq26takww8/nyl+sPO2BsESoVSU8G49U= go.alanpearce.eu/x v0.0.0-20241203124832-a29434dba11a/go.mod h1:FRM6J9HMQ/RV2Q5j+6RKBYWh/YNeEUriGSqDRchiHuQ= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
M gomod2nix.toml → gomod2nix.toml
@@ -152,8 +152,8 @@ [mod."gitlab.com/tozd/go/errors"] version = "v0.10.0" hash = "sha256-oW37KsieVKJOWk9ZXbGuQvuU4nyJCZzgYrTZHFkoCs4=" [mod."go.alanpearce.eu/gomponents"] - version = "v1.3.0" - hash = "sha256-rZz5rJdm58axukN6RlaVKSJ9v2TPngIHt3P1APpXSxY=" + version = "v1.4.0" + hash = "sha256-Q8YN8eNouMnW/JaBJpjUmaOZ5cfBj5gFURgVlbnaiDM=" [mod."go.alanpearce.eu/x"] version = "v0.0.0-20241203124832-a29434dba11a" hash = "sha256-ojqWkz3VqeAOevFxOTO5S3acRItCA4pUrTaul887+x8="
M internal/components/combined.go → internal/components/combined.go
@@ -36,7 +36,7 @@ ), ), ), TBody( - g.Map(result.Hits, func(hit index.DocumentMatch) g.Node { + g.MapIter(result.Hits, func(hit index.DocumentMatch) g.Node { return Tr( Td( openCombinedDialogLink(nix.GetKey(hit.Data)),
M internal/components/options.go → internal/components/options.go
@@ -21,7 +21,7 @@ ), ), ), TBody( - g.Map(result.Hits, func(hit index.DocumentMatch) g.Node { + g.MapIter(result.Hits, func(hit index.DocumentMatch) g.Node { if m := convertMatch[nix.Option](hit.Data); m != nil { return optionRow(hit, *m) }
M internal/components/packages.go → internal/components/packages.go
@@ -22,7 +22,7 @@ ), ), ), TBody( - g.Map(result.Hits, func(hit index.DocumentMatch) g.Node { + g.MapIter(result.Hits, func(hit index.DocumentMatch) g.Node { if m := convertMatch[nix.Package](hit.Data); m != nil { return packageRow(hit, *m) }
M internal/index/search.go → internal/index/search.go
@@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/gob" + "iter" "time" "go.alanpearce.eu/searchix/internal/config" @@ -25,7 +26,7 @@ } type Result struct { *bleve.SearchResult - Hits []DocumentMatch + Hits iter.Seq[DocumentMatch] } type ReadIndex struct { @@ -87,24 +88,31 @@ if err != nil { return nil, errors.WithMessage(err, "failed to execute search query") } - results := make([]DocumentMatch, bleveResult.Hits.Len()) - var buf bytes.Buffer - for i, result := range bleveResult.Hits { - results[i].DocumentMatch = bleveResult.Hits[i] - _, err = buf.WriteString(result.Fields["_data"].(string)) - if err != nil { - return nil, errors.WithMessage(err, "error fetching result data") + hits := func(yield func(DocumentMatch) bool) { + var buf bytes.Buffer + for _, match := range bleveResult.Hits { + hit := DocumentMatch{ + DocumentMatch: match, + Data: nil, + } + _, err := buf.WriteString(match.Fields["_data"].(string)) + if err != nil { + index.log.Warn("error fetching result data", "error", err) + } + err = gob.NewDecoder(&buf).Decode(&hit.Data) + if err != nil { + index.log.Warn("error decoding gob data", "error", err, "data", buf.String()) + } + buf.Reset() + if !yield(hit) { + return + } } - err = gob.NewDecoder(&buf).Decode(&results[i].Data) - if err != nil { - return nil, errors.WithMessagef(err, "error decoding gob data: %s", buf.String()) - } - buf.Reset() } return &Result{ SearchResult: bleveResult, - Hits: results, + Hits: hits, }, nil } } @@ -188,6 +196,7 @@ ) (*nix.Importable, errors.E) { key := nix.MakeKey(source, id) query := bleve.NewDocIDQuery([]string{key}) search := bleve.NewSearchRequest(query) + search.Size = 1 result, err := index.search(ctx, search) if err != nil { @@ -198,5 +207,11 @@ if result.Total == 0 { return nil, nil } - return &result.Hits[0].Data, err + for hit := range result.Hits { + if hit.ID == key { + return &hit.Data, err + } + } + + return nil, err }