all repos — searchix @ dbe952d1df63522e5c32b4ef3e59c43886ba9257

Search engine for NixOS, nix-darwin, home-manager and NUR users

refactor: move sources input data to $data_root/sources
Alan Pearce alan@alanpearce.eu
Sat, 11 May 2024 14:41:00 +0200
commit

dbe952d1df63522e5c32b4ef3e59c43886ba9257

parent

1aa991ae1f1a426424549c92060b204114c8b3c2

2 files changed, 27 insertions(+), 3 deletions(-)

jump to
M import/main.goimport/main.go
@@ -64,11 +64,13 @@ for name, source := range cfg.Sources { 		logger := slog.With("name", name, "importer", source.Type.String())
 		logger.Debug("starting importer")
 
+		importerDataPath := path.Join(cfg.DataPath, "sources", source.Channel)
+
 		switch source.Type {
 		case importer.ChannelNixpkgs:
-			imp = importer.NewNixpkgsChannelImporter(source, cfg.DataPath, logger)
+			imp = importer.NewNixpkgsChannelImporter(source, importerDataPath, logger)
 		case importer.Channel:
-			imp = importer.NewChannelImporter(source, cfg.DataPath, logger)
+			imp = importer.NewChannelImporter(source, importerDataPath, logger)
 		default:
 			log.Printf("unsupported importer type %s", source.Type.String())
 
M internal/search/indexer.gointernal/search/indexer.go
@@ -4,12 +4,14 @@ import ( 	"bytes"
 	"context"
 	"encoding/gob"
+	"io/fs"
 	"log"
 	"log/slog"
 	"os"
 	"path"
 	"searchix/internal/file"
 	"searchix/internal/options"
+	"slices"
 
 	"github.com/blevesearch/bleve/v2"
 	"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
@@ -125,6 +127,11 @@ } 
 const indexBaseName = "index.bleve"
 
+var expectedDataFiles = []string{
+	indexBaseName,
+	"sources",
+}
+
 func NewIndexer(dataRoot string, force bool) (*WriteIndex, error) {
 	var err error
 	bleve.SetLog(log.Default())
@@ -143,7 +150,22 @@ 	var idx bleve.Index
 	if !exists || force {
 		if force {
-			err := os.RemoveAll(indexPath)
+			dir, err := os.ReadDir(dataRoot)
+			if err != nil {
+				return nil, errors.WithMessagef(err, "could not read data directory %s", dataRoot)
+			}
+			remainingFiles := slices.DeleteFunc(dir, func(e fs.DirEntry) bool {
+				return slices.Contains(expectedDataFiles, e.Name())
+			})
+			if len(remainingFiles) > 0 {
+				return nil, errors.Errorf(
+					"cowardly refusing to remove data directory %s as it contains unknown files: %v",
+					dataRoot,
+					remainingFiles,
+				)
+			}
+
+			err = os.RemoveAll(dataRoot)
 			if err != nil {
 				return nil, errors.WithMessagef(err, "could not remove index file %s", indexPath)
 			}