feat: automatically add/remove sources to/from index

2 files changed, 54 insertions, 0 deletions

diff --git a/internal/index/indexer.go b/internal/index/indexer.go
index a661b61..4a6a9d8 100644
--- a/internal/index/indexer.go
+++ b/internal/index/indexer.go
@@ -7,6 +7,7 @@ import (
 	"io/fs"
 	"log"
 	"log/slog"
+	"math"
 	"os"
 	"path"
 	"searchix/internal/file"
@@ -326,3 +327,37 @@ func (i *WriteIndex) Close() error {
 
 	return nil
 }
+
+func (i *WriteIndex) DeleteBySource(source string) error {
+	query := bleve.NewTermQuery(source)
+	search := bleve.NewSearchRequest(query)
+	search.Size = math.MaxInt
+	search.Fields = []string{"_id"}
+
+	results, err := i.index.Search(search)
+	if err != nil {
+		return errors.WithMessagef(err, "failed to query documents of retired index %s", source)
+	}
+
+	batch := i.index.NewBatch()
+	var k uint
+	for _, hit := range results.Hits {
+		batch.Delete(hit.ID)
+		if k++; k%batchSize == 0 {
+			err := i.Flush(batch)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	err = i.Flush(batch)
+	if err != nil {
+		return err
+	}
+
+	if uint64(search.Size) < results.Total {
+		return i.DeleteBySource(source) // unlikely :^)
+	}
+
+	return nil
+}
diff --git a/internal/index/search.go b/internal/index/search.go
index 5c18edb..0b20063 100644
--- a/internal/index/search.go
+++ b/internal/index/search.go
@@ -29,6 +29,25 @@ type ReadIndex struct {
 	meta  *Meta
 }
 
+func (index *ReadIndex) GetEnabledSources() ([]string, error) {
+	facet := bleve.NewFacetRequest("Source", 100)
+	query := bleve.NewMatchAllQuery()
+	search := bleve.NewSearchRequest(query)
+	search.AddFacet("Source", facet)
+
+	results, err := index.index.Search(search)
+	if err != nil {
+		return nil, errors.WithMessage(err, "could not get list of enabled sources from index")
+	}
+
+	enabledSources := make([]string, results.Facets["Source"].Terms.Len())
+	for i, term := range results.Facets["Source"].Terms.Terms() {
+		enabledSources[i] = term.Term
+	}
+
+	return enabledSources, nil
+}
+
 func (index *ReadIndex) GetSource(ctx context.Context, name string) (*bleve.SearchResult, error) {
 	query := bleve.NewTermQuery(name)
 	query.SetField("Source")