refactor: move SetupIndex and indexing progress to importer

4 files changed, 139 insertions, 49 deletions

diff --git a/internal/components/data.go b/internal/components/data.go
index 9bc0c5e..977b90e 100644
--- a/internal/components/data.go
+++ b/internal/components/data.go
@@ -1,21 +1,12 @@
 package components
 
 import (
-	"time"
-
 	"go.alanpearce.eu/searchix/frontend"
 	"go.alanpearce.eu/searchix/internal/config"
 	search "go.alanpearce.eu/searchix/internal/index"
 	"go.alanpearce.eu/searchix/internal/nix"
 )
 
-var Indexing struct {
-	InProgress bool
-	StartedAt  time.Time
-	FinishedAt time.Time
-	NextRun    time.Time
-}
-
 type TemplateData struct {
 	Sources       []*config.Source
 	Source        *config.Source
@@ -43,22 +34,3 @@ func convertMatch[I nix.Importable](m nix.Importable) *I {
 
 	return &i
 }
-
-func SetNextRun(nextRun time.Time) {
-	Indexing.NextRun = nextRun
-}
-
-func SetLastUpdated(last time.Time) {
-	Indexing.FinishedAt = last
-}
-
-func MarkIndexingStarted() {
-	Indexing.StartedAt = time.Now()
-	Indexing.InProgress = true
-}
-
-func MarkIndexingFinished(nextRun time.Time) {
-	Indexing.FinishedAt = time.Now()
-	Indexing.InProgress = false
-	Indexing.NextRun = nextRun
-}
diff --git a/internal/components/search.go b/internal/components/search.go
index 3db1cd4..05cdaa3 100644
--- a/internal/components/search.go
+++ b/internal/components/search.go
@@ -6,6 +6,7 @@ import (
 	g "go.alanpearce.eu/gomponents"
 	. "go.alanpearce.eu/gomponents/html"
 	"go.alanpearce.eu/searchix/internal/config"
+	"go.alanpearce.eu/searchix/internal/importer"
 )
 
 func SearchForm(tdata TemplateData, r ResultData) g.Node {
@@ -41,38 +42,38 @@ func SearchPage(tdata TemplateData, r ResultData, children ...g.Node) g.Node {
 				return A(Href(source.Repo.String()), g.Text(source.Name))
 			}),
 		),
-		g.If(Indexing.InProgress,
+		g.If(importer.Job.InProgress,
 			P(Class("notice"),
-				g.Text("Indexing in progress, started "),
+				g.Text("importer.Indexing in progress, started "),
 				Time(
-					DateTime(Indexing.StartedAt.Format(time.RFC3339)),
-					Title(Indexing.StartedAt.Format(time.DateTime)),
-					g.Text(time.Since(Indexing.StartedAt).Round(time.Second).String()),
+					DateTime(importer.Job.StartedAt.Format(time.RFC3339)),
+					Title(importer.Job.StartedAt.Format(time.DateTime)),
+					g.Text(time.Since(importer.Job.StartedAt).Round(time.Second).String()),
 				),
 				g.Text(" ago. "),
-				g.If(!Indexing.FinishedAt.IsZero(),
+				g.If(!importer.Job.FinishedAt.IsZero(),
 					g.Group([]g.Node{
 						g.Text("Last run took "),
 						Time(
-							DateTime(Indexing.FinishedAt.Format(time.RFC3339)),
-							Title(Indexing.FinishedAt.Format(time.DateTime)),
-							g.Text(time.Since(Indexing.FinishedAt).Round(time.Minute).String()),
+							DateTime(importer.Job.FinishedAt.Format(time.RFC3339)),
+							Title(importer.Job.FinishedAt.Format(time.DateTime)),
+							g.Text(time.Since(importer.Job.FinishedAt).Round(time.Minute).String()),
 						),
 					}),
 				),
 			),
 			P(
-				g.Text("Indexing last ran "),
+				g.Text("importer.Indexing last ran "),
 				Time(
-					DateTime(Indexing.FinishedAt.Format(time.RFC3339)),
-					Title(Indexing.FinishedAt.Format(time.DateTime)),
-					g.Textf("%.0f hours ago", time.Since(Indexing.FinishedAt).Hours()),
+					DateTime(importer.Job.FinishedAt.Format(time.RFC3339)),
+					Title(importer.Job.FinishedAt.Format(time.DateTime)),
+					g.Textf("%.0f hours ago", time.Since(importer.Job.FinishedAt).Hours()),
 				),
 				g.Text(", will run again in "),
 				Time(
-					DateTime(Indexing.NextRun.Format(time.RFC3339)),
-					Title(Indexing.NextRun.Format(time.DateTime)),
-					g.Textf("%.0f hours", time.Until(Indexing.NextRun).Hours()),
+					DateTime(importer.Job.NextRun.Format(time.RFC3339)),
+					Title(importer.Job.NextRun.Format(time.DateTime)),
+					g.Textf("%.0f hours", time.Until(importer.Job.NextRun).Hours()),
 				),
 				g.Text("."),
 			),
diff --git a/internal/importer/main.go b/internal/importer/main.go
index e2c222c..172c504 100644
--- a/internal/importer/main.go
+++ b/internal/importer/main.go
@@ -17,6 +17,39 @@ import (
 	"gitlab.com/tozd/go/errors"
 )
 
+type Options struct {
+	Update    bool
+	Replace   bool
+	LowMemory bool
+	Logger    *log.Logger
+}
+
+var Job struct {
+	InProgress bool
+	StartedAt  time.Time
+	FinishedAt time.Time
+	NextRun    time.Time
+}
+
+func SetNextRun(nextRun time.Time) {
+	Job.NextRun = nextRun
+}
+
+func SetLastUpdated(last time.Time) {
+	Job.FinishedAt = last
+}
+
+func MarkIndexingStarted() {
+	Job.StartedAt = time.Now()
+	Job.InProgress = true
+}
+
+func MarkIndexingFinished(nextRun time.Time) {
+	Job.FinishedAt = time.Now()
+	Job.InProgress = false
+	Job.NextRun = nextRun
+}
+
 func createSourceImporter(
 	parent context.Context,
 	log *log.Logger,
@@ -203,3 +236,87 @@ func (imp *Importer) Start(
 
 	return nil
 }
+
+func SetupIndex(ctx context.Context, cfg *config.Config, options *Options) errors.E {
+	var i uint
+	cfgEnabledSources := make([]string, len(cfg.Importer.Sources))
+	for key := range cfg.Importer.Sources {
+		cfgEnabledSources[i] = key
+		i++
+	}
+	slices.Sort(cfgEnabledSources)
+
+	read, write, exists, err := index.OpenOrCreate(
+		cfg.DataPath,
+		options.Replace,
+		&index.Options{
+			LowMemory: options.LowMemory,
+			Logger:    options.Logger.Named("index"),
+		},
+	)
+	if err != nil {
+		return errors.Wrap(err, "Failed to open or create index")
+	}
+
+	if !exists || options.Replace || options.Update {
+		options.Logger.Info(
+			"Starting build job",
+			"new",
+			!exists,
+			"replace",
+			options.Replace,
+			"update",
+			options.Update,
+		)
+		imp := New(cfg, options.Logger.Named("importer"), write)
+		err = imp.Start(
+			ctx,
+			options.Replace || options.Update,
+			nil,
+		)
+		if err != nil {
+			return errors.Wrap(err, "Failed to build index")
+		}
+		if options.Replace || options.Update {
+			return nil
+		}
+	} else {
+		indexedSources, err := read.GetEnabledSources()
+		if err != nil {
+			return errors.Wrap(err, "Failed to get enabled sources from index")
+		}
+		slices.Sort(indexedSources)
+		if !slices.Equal(cfgEnabledSources, indexedSources) {
+			newSources := slices.DeleteFunc(slices.Clone(cfgEnabledSources), func(s string) bool {
+				return slices.Contains(indexedSources, s)
+			})
+			retiredSources := slices.DeleteFunc(slices.Clone(indexedSources), func(s string) bool {
+				return slices.Contains(cfgEnabledSources, s)
+			})
+			if len(newSources) > 0 {
+				options.Logger.Info("adding new sources", "sources", newSources)
+				imp := New(cfg, options.Logger.Named("importer"), write)
+				err := imp.Start(
+					ctx,
+					false,
+					&newSources,
+				)
+				if err != nil {
+					return errors.Wrap(err, "Failed to update index with new sources")
+				}
+			}
+			if len(retiredSources) > 0 {
+				options.Logger.Info("removing retired sources", "sources", retiredSources)
+				for _, s := range retiredSources {
+					err := write.DeleteBySource(s)
+					if err != nil {
+						return errors.Wrapf(err, "Failed to remove retired source %s", s)
+					}
+				}
+			}
+		}
+	}
+	SetLastUpdated(read.LastUpdated())
+
+	return nil
+}
diff --git a/internal/index/indexer.go b/internal/index/indexer.go
index c4032e8..7591aef 100644
--- a/internal/index/indexer.go
+++ b/internal/index/indexer.go
@@ -32,6 +32,11 @@ import (
 	"gitlab.com/tozd/go/errors"
 )
 
+type Options struct {
+	LowMemory bool
+	Logger    *log.Logger
+}
+
 type WriteIndex struct {
 	index bleve.Index
 	log   *log.Logger
@@ -204,11 +209,6 @@ func deleteIndex(dataRoot string) errors.E {
 	return nil
 }
 
-type Options struct {
-	LowMemory bool
-	Logger    *log.Logger
-}
-
 func OpenOrCreate(
 	dataRoot string,
 	force bool,