all repos — searchix @ 9c9aa6b408812248c9f8c29ecc3546ef64947ea8

Search engine for NixOS, nix-darwin, home-manager and NUR users

perf: iterate over results without keeping the set in memory
Alan Pearce alan@alanpearce.eu
Fri, 21 Mar 2025 21:55:00 +0100
commit

9c9aa6b408812248c9f8c29ecc3546ef64947ea8

parent

e8fbdf3bd12c8920a6e9bd84b34e787764b11eaf

M go.modgo.mod
@@ -17,7 +17,7 @@ github.com/pelletier/go-toml/v2 v2.2.3 	github.com/stoewer/go-strcase v1.3.0
 	github.com/yuin/goldmark v1.7.8
 	gitlab.com/tozd/go/errors v0.10.0
-	go.alanpearce.eu/gomponents v1.3.0
+	go.alanpearce.eu/gomponents v1.4.0
 	go.alanpearce.eu/x v0.0.0-20241203124832-a29434dba11a
 	go.uber.org/zap v1.27.0
 	golang.org/x/net v0.33.0
M go.sumgo.sum
@@ -140,8 +140,8 @@ github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic= github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
 gitlab.com/tozd/go/errors v0.10.0 h1:A98kL+gaDvWnY6ZB/u8zP+sYaWsWUGBHeFMtamvW/74=
 gitlab.com/tozd/go/errors v0.10.0/go.mod h1:q3Ugr0C8dCzMEkrzjjlV2qNsm9e0KvqBjwcbcjCpBe4=
-go.alanpearce.eu/gomponents v1.3.0 h1:yfrDWy77KF/ohDLmgNNud1hMR9WWOvCZJlfyO4SSDo4=
-go.alanpearce.eu/gomponents v1.3.0/go.mod h1:uX96UAsHCut1cKMAYVWWxQ9ADt1CAPI8LpyAu0LRQPs=
+go.alanpearce.eu/gomponents v1.4.0 h1:Ibvoce+U0rnPKlDOE+wXDbNniNQL8mYO667+qS5J1Go=
+go.alanpearce.eu/gomponents v1.4.0/go.mod h1:WxD+6FRSvwThQOzV0r6zPAA9CRb41lutUZMSC7r6BRc=
 go.alanpearce.eu/x v0.0.0-20241203124832-a29434dba11a h1:NUv3AzGxwMVSq26takww8/nyl+sPO2BsESoVSU8G49U=
 go.alanpearce.eu/x v0.0.0-20241203124832-a29434dba11a/go.mod h1:FRM6J9HMQ/RV2Q5j+6RKBYWh/YNeEUriGSqDRchiHuQ=
 go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
M gomod2nix.tomlgomod2nix.toml
@@ -152,8 +152,8 @@ [mod."gitlab.com/tozd/go/errors"]     version = "v0.10.0"
     hash = "sha256-oW37KsieVKJOWk9ZXbGuQvuU4nyJCZzgYrTZHFkoCs4="
   [mod."go.alanpearce.eu/gomponents"]
-    version = "v1.3.0"
-    hash = "sha256-rZz5rJdm58axukN6RlaVKSJ9v2TPngIHt3P1APpXSxY="
+    version = "v1.4.0"
+    hash = "sha256-Q8YN8eNouMnW/JaBJpjUmaOZ5cfBj5gFURgVlbnaiDM="
   [mod."go.alanpearce.eu/x"]
     version = "v0.0.0-20241203124832-a29434dba11a"
     hash = "sha256-ojqWkz3VqeAOevFxOTO5S3acRItCA4pUrTaul887+x8="
M internal/components/combined.gointernal/components/combined.go
@@ -36,7 +36,7 @@ ), 			),
 		),
 		TBody(
-			g.Map(result.Hits, func(hit index.DocumentMatch) g.Node {
+			g.MapIter(result.Hits, func(hit index.DocumentMatch) g.Node {
 				return Tr(
 					Td(
 						openCombinedDialogLink(nix.GetKey(hit.Data)),
M internal/components/options.gointernal/components/options.go
@@ -21,7 +21,7 @@ ), 			),
 		),
 		TBody(
-			g.Map(result.Hits, func(hit index.DocumentMatch) g.Node {
+			g.MapIter(result.Hits, func(hit index.DocumentMatch) g.Node {
 				if m := convertMatch[nix.Option](hit.Data); m != nil {
 					return optionRow(hit, *m)
 				}
M internal/components/packages.gointernal/components/packages.go
@@ -22,7 +22,7 @@ ), 			),
 		),
 		TBody(
-			g.Map(result.Hits, func(hit index.DocumentMatch) g.Node {
+			g.MapIter(result.Hits, func(hit index.DocumentMatch) g.Node {
 				if m := convertMatch[nix.Package](hit.Data); m != nil {
 					return packageRow(hit, *m)
 				}
M internal/index/search.gointernal/index/search.go
@@ -4,6 +4,7 @@ import ( 	"bytes"
 	"context"
 	"encoding/gob"
+	"iter"
 	"time"
 
 	"go.alanpearce.eu/searchix/internal/config"
@@ -25,7 +26,7 @@ } 
 type Result struct {
 	*bleve.SearchResult
-	Hits []DocumentMatch
+	Hits iter.Seq[DocumentMatch]
 }
 
 type ReadIndex struct {
@@ -87,24 +88,31 @@ if err != nil { 			return nil, errors.WithMessage(err, "failed to execute search query")
 		}
 
-		results := make([]DocumentMatch, bleveResult.Hits.Len())
-		var buf bytes.Buffer
-		for i, result := range bleveResult.Hits {
-			results[i].DocumentMatch = bleveResult.Hits[i]
-			_, err = buf.WriteString(result.Fields["_data"].(string))
-			if err != nil {
-				return nil, errors.WithMessage(err, "error fetching result data")
+		hits := func(yield func(DocumentMatch) bool) {
+			var buf bytes.Buffer
+			for _, match := range bleveResult.Hits {
+				hit := DocumentMatch{
+					DocumentMatch: match,
+					Data:          nil,
+				}
+				_, err := buf.WriteString(match.Fields["_data"].(string))
+				if err != nil {
+					index.log.Warn("error fetching result data", "error", err)
+				}
+				err = gob.NewDecoder(&buf).Decode(&hit.Data)
+				if err != nil {
+					index.log.Warn("error decoding gob data", "error", err, "data", buf.String())
+				}
+				buf.Reset()
+				if !yield(hit) {
+					return
+				}
 			}
-			err = gob.NewDecoder(&buf).Decode(&results[i].Data)
-			if err != nil {
-				return nil, errors.WithMessagef(err, "error decoding gob data: %s", buf.String())
-			}
-			buf.Reset()
 		}
 
 		return &Result{
 			SearchResult: bleveResult,
-			Hits:         results,
+			Hits:         hits,
 		}, nil
 	}
 }
@@ -188,6 +196,7 @@ ) (*nix.Importable, errors.E) { 	key := nix.MakeKey(source, id)
 	query := bleve.NewDocIDQuery([]string{key})
 	search := bleve.NewSearchRequest(query)
+	search.Size = 1
 
 	result, err := index.search(ctx, search)
 	if err != nil {
@@ -198,5 +207,11 @@ if result.Total == 0 { 		return nil, nil
 	}
 
-	return &result.Hits[0].Data, err
+	for hit := range result.Hits {
+		if hit.ID == key {
+			return &hit.Data, err
+		}
+	}
+
+	return nil, err
 }