about summary refs log tree commit diff stats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore2
-rw-r--r--CHANGELOG.md55
-rw-r--r--cmd/searchix-web/main.go58
-rw-r--r--frontend/static/style.css30
-rw-r--r--internal/components/combined.go4
-rw-r--r--internal/components/data.go28
-rw-r--r--internal/components/markdown.go2
-rw-r--r--internal/components/options.go6
-rw-r--r--internal/components/packages.go6
-rw-r--r--internal/components/search.go40
-rw-r--r--internal/fetcher/buf.go24
-rw-r--r--internal/fetcher/http.go2
-rw-r--r--internal/importer/main.go193
-rw-r--r--internal/importer/main_test.go10
-rw-r--r--internal/index/indexer.go10
-rw-r--r--internal/server/mux.go6
-rw-r--r--justfile13
-rw-r--r--nix/modules/default.nix2
-rw-r--r--nix/package.nix4
-rw-r--r--searchix.go252
-rw-r--r--web/searchix.go60
21 files changed, 453 insertions, 354 deletions
diff --git a/.gitignore b/.gitignore
index 4953924..be8e8fa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,6 @@
 ### https://raw.github.com/github/gitignore/4488915eec0b3a45b5c63ead28f286819c0917de/Go.gitignore
 
 # Binaries for programs and plugins
-/*
 *.exe
 *.exe~
 *.dll
@@ -19,6 +18,7 @@
 # Go workspace file
 go.work
 
+/result
 /.env
 /.envrc
 /.pre-commit-config.yaml
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f6988be..9ccbf64 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,9 +1,60 @@
 # Changelog
 
-## [v0.1.31](https://git.alanpearce.eu/searchix/diff/?id=v0.1.30&id2=d4ec6e5beecd549114dafd0b7c3b4a9d910388fb) (2025-03-21)
+## [v0.1.34](https://git.alanpearce.eu/searchix/diff/?id=v0.1.33&id2=abf819db104560670cdc388a584d09968aac7bbb) (2025-03-24)
 
 ### Features
 
+- buffer fetched HTTP bodies
+  ([abf819d](https://git.alanpearce.eu/searchix/commit/?id=abf819db104560670cdc388a584d09968aac7bbb))
+- re-index on startup if last run > 24 hours ago
+  ([6984d4d](https://git.alanpearce.eu/searchix/commit/?id=6984d4d32ab506494394f8a6a8aa18041c45b9e8))
+
+### Fixes
+
+- another typo in search markup
+  ([efaaf3f](https://git.alanpearce.eu/searchix/commit/?id=efaaf3f5296807eec377b19d4733fd360c8c655f))
+- typo in indexing status
+  ([fecc84f](https://git.alanpearce.eu/searchix/commit/?id=fecc84fab93bb3228be5fee4160820751195feec))
+- nil pointer panic in importer
+  ([0fb48cf](https://git.alanpearce.eu/searchix/commit/?id=0fb48cf6dd934778584120ebbc9e568c8b80b80e))
+
+### [v0.1.33](https://git.alanpearce.eu/searchix/diff/?id=v0.1.32&id2=v0.1.33) (2025-03-24)
+
+#### Fixes
+
+- build failure due to outdated file reference
+  ([dd58d99](https://git.alanpearce.eu/searchix/commit/?id=dd58d9925b68c7cd639c5e217782709c1936497f))
+
+### [v0.1.32](https://git.alanpearce.eu/searchix/diff/?id=v0.1.31&id2=v0.1.32) (2025-03-24)
+
+#### Features
+
+- improve handling of long option descriptions
+  ([4fa3ceb](https://git.alanpearce.eu/searchix/commit/?id=4fa3ceb5ac040a84f2da405c46e4af1231d22f17))
+- widen body
+  ([1c56ff7](https://git.alanpearce.eu/searchix/commit/?id=1c56ff7fc392f2d7992e53add5bd76bc8855d4a6))
+- improve display of indexing times and durations
+  ([297a7a2](https://git.alanpearce.eu/searchix/commit/?id=297a7a2096a12e6ac50d7b2077f2d75a7026fae4))
+- link directly to version changelog in footer
+  ([7d4ff28](https://git.alanpearce.eu/searchix/commit/?id=7d4ff28794fcc8d7dd8c87f300ce6a6235f87cc5))
+- **js:** allow opening details in new tab/window
+  ([03fa0eb](https://git.alanpearce.eu/searchix/commit/?id=03fa0eb00aef448bec05280a121c659a600fc13f))
+- promote prefix/exact name/attr matches
+  ([1328df9](https://git.alanpearce.eu/searchix/commit/?id=1328df9c5ca44e903a052c9bfc9a67dd34e25704))
+
+#### Fixes
+
+- missing content of last index run time
+  ([0dc2635](https://git.alanpearce.eu/searchix/commit/?id=0dc2635e199591b917c8cc8ff40df3c3da6f9ec6))
+- block single-character queries
+  ([0cd9a24](https://git.alanpearce.eu/searchix/commit/?id=0cd9a2446413e9050c24ccd409bc8f97ad18076a))
+- don't attempt to load more than configured number of results
+  ([d39204e](https://git.alanpearce.eu/searchix/commit/?id=d39204eadf673b6c7ae940203fd75c0805245a96))
+
+### [v0.1.31](https://git.alanpearce.eu/searchix/diff/?id=v0.1.30&id2=v0.1.31) (2025-03-22)
+
+#### Features
+
 - **nixos-module:** allow setting environment variables
   ([e8fbdf3](https://git.alanpearce.eu/searchix/commit/?id=e8fbdf3bd12c8920a6e9bd84b34e787764b11eaf))
 - **nixos-module:** allow setting environment variables
@@ -25,7 +76,7 @@
 - Convert templ components to gomponents
   ([896d844](https://git.alanpearce.eu/searchix/commit/?id=896d844cac976afd0ee8aa73dd2fb28e15e7ac79))
 
-### Fixes
+#### Fixes
 
 - remove rendering from search timeout restriction
   ([d4ec6e5](https://git.alanpearce.eu/searchix/commit/?id=d4ec6e5beecd549114dafd0b7c3b4a9d910388fb))
diff --git a/cmd/searchix-web/main.go b/cmd/searchix-web/main.go
index 6d6dffa..20d4a94 100644
--- a/cmd/searchix-web/main.go
+++ b/cmd/searchix-web/main.go
@@ -7,11 +7,15 @@ import (
 	"os"
 	"os/signal"
 	"runtime/pprof"
+	"sync"
 
 	"badc0de.net/pkg/flagutil"
+	"github.com/getsentry/sentry-go"
 
-	"go.alanpearce.eu/searchix"
 	"go.alanpearce.eu/searchix/internal/config"
+	"go.alanpearce.eu/searchix/internal/importer"
+	"go.alanpearce.eu/searchix/internal/index"
+	"go.alanpearce.eu/searchix/web"
 	"go.alanpearce.eu/x/log"
 )
 
@@ -24,7 +28,6 @@ var (
 	)
 	dev        = flag.Bool("dev", false, "enable live reloading and nicer logging")
 	replace    = flag.Bool("replace", false, "replace existing index and exit")
-	update     = flag.Bool("update", false, "update index and exit")
 	version    = flag.Bool("version", false, "print version information")
 	cpuprofile = flag.String("cpuprofile", "", "enable CPU profiling and save to `file`")
 )
@@ -69,35 +72,66 @@ func main() {
 	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer cancel()
 
-	s, err := searchix.New(cfg, logger)
+	read, write, exists, err := index.OpenOrCreate(
+		cfg.DataPath,
+		*replace,
+		&index.Options{
+			LowMemory: cfg.Importer.LowMemory,
+			Logger:    logger.Named("index"),
+		},
+	)
+	if err != nil {
+		logger.Fatal("Failed to open or create index", "error", err)
+	}
+
+	s, err := web.New(cfg, logger, read)
 	if err != nil {
-		logger.Fatal("Failed to initialise searchix", "error", err)
+		logger.Fatal("Failed to initialise searchix-web", "error", err)
 	}
 
-	err = s.SetupIndex(ctx, &searchix.IndexOptions{
-		Update:    *update,
-		Replace:   *replace,
-		LowMemory: cfg.Importer.LowMemory,
-		Logger:    logger,
+	imp, err := importer.New(cfg, &importer.Options{
+		WriteIndex: write,
+		LowMemory:  cfg.Importer.LowMemory,
+		Logger:     logger.Named("importer"),
 	})
 	if err != nil {
-		logger.Fatal("Failed to setup index", "error", err)
+		logger.Fatal("Failed to create importer", "error", err)
 	}
 
-	if *replace || *update {
+	if !exists || *replace {
+		err := imp.Start(ctx, true, nil)
+		if err != nil {
+			logger.Fatal("Failed to start importer", "error", err)
+		}
+
 		return
 	}
 
+	err = imp.EnsureSourcesIndexed(ctx, read)
+	if err != nil {
+		logger.Fatal("Failed to setup index", "error", err)
+	}
+
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
 	go func() {
-		err = s.Start(ctx, *dev)
+		defer wg.Done()
+		err := s.Start(*dev)
 		if err != nil {
 			// Error starting or closing listener:
 			logger.Fatal("error", "error", err)
 		}
 	}()
 
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		imp.StartUpdateTimer(ctx, sentry.CurrentHub().Clone())
+	}()
+
 	<-ctx.Done()
 	logger.Debug("calling stop")
 	s.Stop()
+	wg.Wait()
 	logger.Debug("done")
 }
diff --git a/frontend/static/style.css b/frontend/static/style.css
index 8766844..f7d1f75 100644
--- a/frontend/static/style.css
+++ b/frontend/static/style.css
@@ -2,7 +2,7 @@
   --sans-font: -apple-system, BlinkMacSystemFont, sans-serif;
   --standard-border-radius: 0;
   --preformatted: var(--code);
-  --min-width: 60rem;
+  --min-width: 80rem;
   --accent-error: #ffe0e0;
 }
 
@@ -162,13 +162,39 @@ dialog > h2 {
 }
 
 table {
-  width: 100%;
   margin-top: 0;
+  width: 100%;
+  table-layout: fixed;
+  white-space: nowrap;
+}
+
+th.description {
+  width: 50%;
+}
+
+th.score {
+  width: 8ex;
 }
 
 td,
 th {
   padding: 0.25rem 0.5rem;
+  text-overflow: ellipsis;
+  overflow: clip;
+}
+
+tr {
+  height: 1.5rem;
+}
+
+td.description > * {
+  max-width: 100%;
+  text-overflow: ellipsis;
+  overflow: clip;
+}
+
+td.description > *:not(:first-child) {
+  display: none;
 }
 
 ul:only-child {
diff --git a/internal/components/combined.go b/internal/components/combined.go
index fe97e14..8c2dc34 100644
--- a/internal/components/combined.go
+++ b/internal/components/combined.go
@@ -31,7 +31,7 @@ func Combined(result *index.Result) g.Node {
 				Th(Scope("col"), g.Text("Attribute")),
 				Th(Scope("col"), g.Text("Description")),
 				g.If(config.DevMode,
-					Th(Scope("col"), g.Text("Score")),
+					Th(Scope("col"), Class("score"), g.Text("Score")),
 				),
 			),
 		),
@@ -41,7 +41,7 @@ func Combined(result *index.Result) g.Node {
 					Td(
 						openCombinedDialogLink(nix.GetKey(hit.Data)),
 					),
-					Td(
+					Td(Class("description"),
 						CombinedData(hit.Data),
 					),
 					g.If(config.DevMode,
diff --git a/internal/components/data.go b/internal/components/data.go
index 9bc0c5e..977b90e 100644
--- a/internal/components/data.go
+++ b/internal/components/data.go
@@ -1,21 +1,12 @@
 package components
 
 import (
-	"time"
-
 	"go.alanpearce.eu/searchix/frontend"
 	"go.alanpearce.eu/searchix/internal/config"
 	search "go.alanpearce.eu/searchix/internal/index"
 	"go.alanpearce.eu/searchix/internal/nix"
 )
 
-var Indexing struct {
-	InProgress bool
-	StartedAt  time.Time
-	FinishedAt time.Time
-	NextRun    time.Time
-}
-
 type TemplateData struct {
 	Sources       []*config.Source
 	Source        *config.Source
@@ -43,22 +34,3 @@ func convertMatch[I nix.Importable](m nix.Importable) *I {
 
 	return &i
 }
-
-func SetNextRun(nextRun time.Time) {
-	Indexing.NextRun = nextRun
-}
-
-func SetLastUpdated(last time.Time) {
-	Indexing.FinishedAt = last
-}
-
-func MarkIndexingStarted() {
-	Indexing.StartedAt = time.Now()
-	Indexing.InProgress = true
-}
-
-func MarkIndexingFinished(nextRun time.Time) {
-	Indexing.FinishedAt = time.Now()
-	Indexing.InProgress = false
-	Indexing.NextRun = nextRun
-}
diff --git a/internal/components/markdown.go b/internal/components/markdown.go
index 405ab52..a26fe3d 100644
--- a/internal/components/markdown.go
+++ b/internal/components/markdown.go
@@ -4,7 +4,7 @@ import (
 	"regexp"
 )
 
-var firstSentenceRegexp = regexp.MustCompile(`^.*?\.[[:space:]]`)
+var firstSentenceRegexp = regexp.MustCompile(`^.+?(\.[[:space:]]|:\n)`)
 
 func firstSentence[T ~string](text T) T {
 	if fs := firstSentenceRegexp.FindString(string(text)); fs != "" {
diff --git a/internal/components/options.go b/internal/components/options.go
index 1d01784..630fecd 100644
--- a/internal/components/options.go
+++ b/internal/components/options.go
@@ -14,9 +14,9 @@ func Options(result *index.Result) g.Node {
 		THead(
 			Tr(
 				Th(Scope("col"), g.Text("Title")),
-				Th(Scope("col"), g.Text("Description")),
+				Th(Scope("col"), Class("description"), g.Text("Description")),
 				g.If(config.DevMode,
-					Th(Scope("col"), g.Text("Score")),
+					Th(Scope("col"), Class("score"), g.Text("Score")),
 				),
 			),
 		),
@@ -37,7 +37,7 @@ func optionRow(hit index.DocumentMatch, o nix.Option) g.Node {
 		Td(
 			openDialogLink(o.Name),
 		),
-		Td(
+		Td(Class("description"),
 			firstSentence(o.Description),
 			Dialog(ID(o.Name)),
 		),
diff --git a/internal/components/packages.go b/internal/components/packages.go
index db45302..90bf92d 100644
--- a/internal/components/packages.go
+++ b/internal/components/packages.go
@@ -15,9 +15,9 @@ func Packages(result *index.Result) g.Node {
 			Tr(
 				Th(Scope("col"), g.Text("Attribute")),
 				Th(Scope("col"), g.Text("Name")),
-				Th(Scope("col"), g.Text("Description")),
+				Th(Scope("col"), Class("description"), g.Text("Description")),
 				g.If(config.DevMode,
-					Th(Scope("col"), g.Text("Score")),
+					Th(Scope("col"), Class("score"), g.Text("Score")),
 				),
 			),
 		),
@@ -41,7 +41,7 @@ func packageRow(hit index.DocumentMatch, p nix.Package) g.Node {
 		Td(
 			g.Text(p.Name),
 		),
-		Td(
+		Td(Class("description"),
 			g.Text(p.Description),
 		),
 		g.If(config.DevMode,
diff --git a/internal/components/search.go b/internal/components/search.go
index b4ef7bd..e68d0fc 100644
--- a/internal/components/search.go
+++ b/internal/components/search.go
@@ -6,6 +6,7 @@ import (
 	g "go.alanpearce.eu/gomponents"
 	. "go.alanpearce.eu/gomponents/html"
 	"go.alanpearce.eu/searchix/internal/config"
+	"go.alanpearce.eu/searchix/internal/importer"
 )
 
 func SearchForm(tdata TemplateData, r ResultData) g.Node {
@@ -19,6 +20,7 @@ func SearchForm(tdata TemplateData, r ResultData) g.Node {
 			Input(
 				ID("query"),
 				Aria("labelledby", "legend"),
+				MinLength("2"),
 				Name("query"),
 				Type("search"),
 				Value(r.Query),
@@ -40,36 +42,38 @@ func SearchPage(tdata TemplateData, r ResultData, children ...g.Node) g.Node {
 				return A(Href(source.Repo.String()), g.Text(source.Name))
 			}),
 		),
-		g.If(Indexing.InProgress,
+		g.If(importer.Job.InProgress,
 			P(Class("notice"),
 				g.Text("Indexing in progress, started "),
 				Time(
-					DateTime(Indexing.StartedAt.Format(time.RFC3339)),
-					Title(Indexing.StartedAt.Format(time.RFC3339)),
-					g.Text(time.Since(Indexing.StartedAt).Round(time.Second).String()),
+					DateTime(importer.Job.StartedAt.Format(time.RFC3339)),
+					Title(importer.Job.StartedAt.Format(time.DateTime)),
+					g.Text(time.Since(importer.Job.StartedAt).Round(time.Second).String()),
 				),
 				g.Text(" ago. "),
-				g.If(!Indexing.FinishedAt.IsZero(),
-					g.Text("Last run took "),
-					Time(
-						DateTime(Indexing.FinishedAt.Format(time.RFC3339)),
-						Title(Indexing.FinishedAt.Format(time.RFC3339)),
-						g.Text(time.Since(Indexing.FinishedAt).Round(time.Minute).String()),
-					),
+				g.If(!importer.Job.FinishedAt.IsZero(),
+					g.Group([]g.Node{
+						g.Text("Last run took "),
+						Time(
+							DateTime(importer.Job.FinishedAt.Format(time.RFC3339)),
+							Title(importer.Job.FinishedAt.Format(time.DateTime)),
+							g.Text(time.Since(importer.Job.FinishedAt).Round(time.Minute).String()),
+						),
+					}),
 				),
 			),
 			P(
 				g.Text("Indexing last ran "),
 				Time(
-					DateTime(Indexing.FinishedAt.Format(time.RFC3339)),
-					Title(Indexing.FinishedAt.Format(time.RFC3339)),
-					g.Text(time.Since(Indexing.FinishedAt).Round(time.Minute).String()),
+					DateTime(importer.Job.FinishedAt.Format(time.RFC3339)),
+					Title(importer.Job.FinishedAt.Format(time.DateTime)),
+					g.Textf("%.0f hours ago", time.Since(importer.Job.FinishedAt).Hours()),
 				),
-				g.Text(" ago, will run again in "),
+				g.Text(", will run again in "),
 				Time(
-					DateTime(Indexing.NextRun.Format(time.RFC3339)),
-					Title(Indexing.NextRun.Format(time.RFC3339)),
-					g.Text(time.Until(Indexing.NextRun).Round(time.Minute).String()),
+					DateTime(importer.Job.NextRun.Format(time.RFC3339)),
+					Title(importer.Job.NextRun.Format(time.DateTime)),
+					g.Textf("%.0f hours", time.Until(importer.Job.NextRun).Hours()),
 				),
 				g.Text("."),
 			),
diff --git a/internal/fetcher/buf.go b/internal/fetcher/buf.go
new file mode 100644
index 0000000..6572130
--- /dev/null
+++ b/internal/fetcher/buf.go
@@ -0,0 +1,24 @@
+package fetcher
+
+import (
+	"bufio"
+	"io"
+
+	"gitlab.com/tozd/go/errors"
+)
+
+type Reader struct {
+	closer io.ReadCloser
+	*bufio.Reader
+}
+
+func NewReadCloser(body io.ReadCloser) *Reader {
+	return &Reader{
+		closer: body,
+		Reader: bufio.NewReader(body),
+	}
+}
+
+func (r *Reader) Close() error {
+	return errors.WithStack(r.closer.Close())
+}
diff --git a/internal/fetcher/http.go b/internal/fetcher/http.go
index ba99c3a..b0768ab 100644
--- a/internal/fetcher/http.go
+++ b/internal/fetcher/http.go
@@ -94,5 +94,5 @@ func fetchFileIfNeeded(
 		err = errors.Errorf("got response code %d, don't know what to do", res.StatusCode)
 	}
 
-	return body, newMtime, err
+	return NewReadCloser(body), newMtime, err
 }
diff --git a/internal/importer/main.go b/internal/importer/main.go
index e2c222c..184c6df 100644
--- a/internal/importer/main.go
+++ b/internal/importer/main.go
@@ -3,11 +3,14 @@ package importer
 import (
 	"context"
 	"fmt"
+	"maps"
+	"math"
 	"os/exec"
 	"slices"
 	"strings"
 	"time"
 
+	"github.com/getsentry/sentry-go"
 	"go.alanpearce.eu/searchix/internal/config"
 	"go.alanpearce.eu/searchix/internal/fetcher"
 	"go.alanpearce.eu/searchix/internal/index"
@@ -17,11 +20,43 @@ import (
 	"gitlab.com/tozd/go/errors"
 )
 
+type Options struct {
+	LowMemory  bool
+	Logger     *log.Logger
+	WriteIndex *index.WriteIndex
+}
+
+var Job struct {
+	InProgress bool
+	StartedAt  time.Time
+	FinishedAt time.Time
+	NextRun    time.Time
+}
+
+func SetNextRun(nextRun time.Time) {
+	Job.NextRun = nextRun
+}
+
+func SetLastUpdated(last time.Time) {
+	Job.FinishedAt = last
+}
+
+func MarkIndexingStarted() {
+	Job.StartedAt = time.Now()
+	Job.InProgress = true
+}
+
+func MarkIndexingFinished(nextRun time.Time) {
+	Job.FinishedAt = time.Now()
+	Job.InProgress = false
+	Job.NextRun = nextRun
+}
+
 func createSourceImporter(
 	parent context.Context,
 	log *log.Logger,
 	meta *index.Meta,
-	indexer *index.WriteIndex,
+	writeIndex *index.WriteIndex,
 	forceUpdate bool,
 ) func(*config.Source) errors.E {
 	return func(source *config.Source) errors.E {
@@ -124,7 +159,7 @@ func createSourceImporter(
 				return errors.WithMessagef(err, "failed to create processor")
 			}
 
-			hadWarnings, err := process(ctx, indexer, processor, logger)
+			hadWarnings, err := process(ctx, writeIndex, processor, logger)
 			if err != nil {
 				return errors.WithMessagef(err, "failed to process source")
 			}
@@ -149,18 +184,6 @@ type Importer struct {
 	indexer *index.WriteIndex
 }
 
-func New(
-	cfg *config.Config,
-	log *log.Logger,
-	indexer *index.WriteIndex,
-) *Importer {
-	return &Importer{
-		config:  cfg,
-		log:     log,
-		indexer: indexer,
-	}
-}
-
 func (imp *Importer) Start(
 	ctx context.Context,
 	forceUpdate bool,
@@ -201,5 +224,147 @@ func (imp *Importer) Start(
 		return errors.Wrap(err, "failed to save metadata")
 	}
 
+	SetLastUpdated(time.Now())
+
 	return nil
 }
+
+func New(
+	cfg *config.Config,
+	options *Options,
+) (*Importer, errors.E) {
+
+	return &Importer{
+		config:  cfg,
+		log:     options.Logger,
+		indexer: options.WriteIndex,
+	}, nil
+}
+
+func (imp *Importer) EnsureSourcesIndexed(
+	ctx context.Context,
+	read *index.ReadIndex,
+) errors.E {
+	cfgEnabledSources := slices.Collect(maps.Keys(imp.config.Importer.Sources))
+	slices.Sort(cfgEnabledSources)
+
+	indexedSources, err := read.GetEnabledSources()
+	if err != nil {
+		return errors.Wrap(err, "Failed to get enabled sources from index")
+	}
+	slices.Sort(indexedSources)
+	if !slices.Equal(cfgEnabledSources, indexedSources) {
+		newSources := slices.DeleteFunc(slices.Clone(cfgEnabledSources), func(s string) bool {
+			return slices.Contains(indexedSources, s)
+		})
+		retiredSources := slices.DeleteFunc(slices.Clone(indexedSources), func(s string) bool {
+			return slices.Contains(cfgEnabledSources, s)
+		})
+		if len(newSources) > 0 {
+			imp.log.Info("adding new sources", "sources", newSources)
+			err := imp.Start(
+				ctx,
+				false,
+				&newSources,
+			)
+			if err != nil {
+				return errors.Wrap(err, "Failed to update index with new sources")
+			}
+		}
+		if len(retiredSources) > 0 {
+			imp.log.Info("removing retired sources", "sources", retiredSources)
+			for _, s := range retiredSources {
+				err := imp.indexer.DeleteBySource(s)
+				if err != nil {
+					return errors.Wrapf(err, "Failed to remove retired source %s", s)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (imp *Importer) StartUpdateTimer(
+	ctx context.Context,
+	localHub *sentry.Hub,
+) {
+	const monitorSlug = "import"
+	localHub.WithScope(func(scope *sentry.Scope) {
+		var err errors.E
+		scope.SetContext("monitor", sentry.Context{"slug": monitorSlug})
+		monitorConfig := &sentry.MonitorConfig{
+			Schedule:      sentry.IntervalSchedule(1, sentry.MonitorScheduleUnitDay),
+			MaxRuntime:    int64(math.Ceil(imp.config.Importer.Timeout.Minutes())),
+			CheckInMargin: 5,
+			Timezone:      time.Local.String(),
+		}
+
+		var nextRun time.Time
+		if Job.FinishedAt.Before(time.Now().Add(-24 * time.Hour)) {
+			imp.log.Info("indexing last ran more than 24 hours ago, scheduling immediate update")
+			nextRun = time.Now()
+		} else {
+			nextRun = nextUTCOccurrenceOfTime(imp.config.Importer.UpdateAt)
+		}
+		SetNextRun(nextRun)
+		for {
+			imp.log.Debug("scheduling next run", "next-run", nextRun)
+			select {
+			case <-ctx.Done():
+				imp.log.Debug("stopping scheduler")
+
+				return
+			case <-time.After(time.Until(nextRun)):
+			}
+			imp.log.Info("updating index")
+
+			eventID := localHub.CaptureCheckIn(&sentry.CheckIn{
+				MonitorSlug: monitorSlug,
+				Status:      sentry.CheckInStatusInProgress,
+			}, monitorConfig)
+			MarkIndexingStarted()
+
+			err = imp.Start(ctx, false, nil)
+			if err != nil {
+				imp.log.Warn("error updating index", "error", err)
+
+				localHub.CaptureException(err)
+				localHub.CaptureCheckIn(&sentry.CheckIn{
+					ID:          *eventID,
+					MonitorSlug: monitorSlug,
+					Status:      sentry.CheckInStatusError,
+				}, monitorConfig)
+			} else {
+				imp.log.Info("update complete")
+
+				localHub.CaptureCheckIn(&sentry.CheckIn{
+					ID:          *eventID,
+					MonitorSlug: monitorSlug,
+					Status:      sentry.CheckInStatusOK,
+				}, monitorConfig)
+			}
+			nextRun = nextUTCOccurrenceOfTime(imp.config.Importer.UpdateAt)
+			MarkIndexingFinished(nextRun)
+		}
+	})
+}
+
+func nextUTCOccurrenceOfTime(dayTime config.LocalTime) time.Time {
+	now := time.Now()
+	nextRun := time.Date(
+		now.Year(),
+		now.Month(),
+		now.Day(),
+		dayTime.Hour,
+		dayTime.Minute,
+		dayTime.Second,
+		0,
+		time.UTC,
+	)
+	if nextRun.Before(now) {
+		return nextRun.AddDate(0, 0, 1)
+	}
+
+	return nextRun
+}
diff --git a/internal/importer/main_test.go b/internal/importer/main_test.go
index 576d681..84f6adf 100644
--- a/internal/importer/main_test.go
+++ b/internal/importer/main_test.go
@@ -22,7 +22,15 @@ func BenchmarkImporterLowMemory(b *testing.B) {
 		b.Fatal(err)
 	}
 
-	imp := New(&cfg, logger.Named("importer"), write)
+	imp, err := New(&cfg, &Options{
+		Logger:     logger.Named("importer"),
+		LowMemory:  true,
+		WriteIndex: write,
+	})
+	if err != nil {
+		b.Fatal(err)
+	}
+
 	err = imp.Start(
 		context.Background(),
 		false,
diff --git a/internal/index/indexer.go b/internal/index/indexer.go
index c4032e8..7591aef 100644
--- a/internal/index/indexer.go
+++ b/internal/index/indexer.go
@@ -32,6 +32,11 @@ import (
 	"gitlab.com/tozd/go/errors"
 )
 
+type Options struct {
+	LowMemory bool
+	Logger    *log.Logger
+}
+
 type WriteIndex struct {
 	index bleve.Index
 	log   *log.Logger
@@ -204,11 +209,6 @@ func deleteIndex(dataRoot string) errors.E {
 	return nil
 }
 
-type Options struct {
-	LowMemory bool
-	Logger    *log.Logger
-}
-
 func OpenOrCreate(
 	dataRoot string,
 	force bool,
diff --git a/internal/server/mux.go b/internal/server/mux.go
index 151fb96..968b37c 100644
--- a/internal/server/mux.go
+++ b/internal/server/mux.go
@@ -96,6 +96,12 @@ func NewMux(
 			if r.URL.Query().Has("query") {
 				qs := r.URL.Query().Get("query")
 
+				if len(qs) < 2 {
+					errorHandler(w, r, "Query too short", http.StatusBadRequest)
+
+					return
+				}
+
 				var pageSize int = search.DefaultPageSize
 				var pageNumber = 1
 				if pg := r.URL.Query().Get("page"); pg != "" {
diff --git a/justfile b/justfile
index 200457b..734cb1f 100644
--- a/justfile
+++ b/justfile
@@ -2,7 +2,7 @@ default:
 	@just --list --justfile {{ justfile() }} --unsorted
 
 prepare:
-	nix build .#css -o frontend/static/base.css
+	cp -f $(nix build --print-out-paths .#css) frontend/static/base.css
 
 update-go-dependencies:
 	go get -u all
@@ -51,9 +51,13 @@ update-nix-package-version:
 	sd 'h=v.+$' "h=v$VER" nix/modules/default.nix
 
 release: clean-workdir changelog update-nix-package-version
+	#!/bin/sh
+	set -eu
+	VERSION=$(convco version --bump --print-prefix)
 	git add CHANGELOG.md nix/package.nix nix/modules/default.nix
-	git commit -m "chore: release $(convco version --bump)"
-	git tag $(convco version --bump --print-prefix)
+	git commit -m "chore: release ${VERSION}"
+	git tag "${VERSION}"
+	git push origin main "${VERSION}"
 
 run: dev
 
@@ -62,6 +66,3 @@ dev:
 
 reindex *flags:
 	wgo run --exit ./cmd/searchix-web --config config.toml --replace --dev {{ flags }}
-
-update *flags:
-	wgo run --exit ./cmd/searchix-web --config config.toml --update --dev {{ flags }}
diff --git a/nix/modules/default.nix b/nix/modules/default.nix
index 5184dde..3aba4da 100644
--- a/nix/modules/default.nix
+++ b/nix/modules/default.nix
@@ -200,7 +200,7 @@ in
       description = ''
         Configuration for searchix.
 
-        See https://git.alanpearce.eu/searchix/tree/defaults.toml?h=v0.1.31
+        See https://git.alanpearce.eu/searchix/tree/defaults.toml?h=v0.1.34
       '';
     };
   };
diff --git a/nix/package.nix b/nix/package.nix
index 996c0c4..c2c1054 100644
--- a/nix/package.nix
+++ b/nix/package.nix
@@ -13,7 +13,7 @@
 , css
 }:
 let
-  version = "0.1.31";
+  version = "0.1.34";
 in
 buildGoApplication {
   pname = "searchix";
@@ -24,10 +24,10 @@ buildGoApplication {
       (unions [
         ../go.mod
         ../go.sum
-        ../searchix.go
         ../internal
         ../frontend
         ../cmd
+        ../web
       ])
       (maybeMissing ../frontend/static/base.css);
   };
diff --git a/searchix.go b/searchix.go
deleted file mode 100644
index 8bd696d..0000000
--- a/searchix.go
+++ /dev/null
@@ -1,252 +0,0 @@
-package searchix
-
-import (
-	"context"
-	"math"
-	"slices"
-	"sync"
-	"time"
-
-	"go.alanpearce.eu/searchix/internal/components"
-	"go.alanpearce.eu/searchix/internal/config"
-	"go.alanpearce.eu/searchix/internal/importer"
-	"go.alanpearce.eu/searchix/internal/index"
-	"go.alanpearce.eu/searchix/internal/server"
-	"go.alanpearce.eu/x/log"
-
-	"github.com/getsentry/sentry-go"
-	"gitlab.com/tozd/go/errors"
-)
-
-func nextUTCOccurrenceOfTime(t config.LocalTime) time.Time {
-	now := time.Now()
-	dayTime := t
-	nextRun := time.Date(
-		now.Year(),
-		now.Month(),
-		now.Day(),
-		dayTime.Hour,
-		dayTime.Minute,
-		dayTime.Second,
-		0,
-		time.UTC,
-	)
-	if nextRun.Before(now) {
-		return nextRun.AddDate(0, 0, 1)
-	}
-
-	return nextRun
-}
-
-type IndexOptions struct {
-	Update    bool
-	Replace   bool
-	LowMemory bool
-	Logger    *log.Logger
-}
-
-func (s *Server) SetupIndex(ctx context.Context, options *IndexOptions) errors.E {
-	var i uint
-	cfgEnabledSources := make([]string, len(s.cfg.Importer.Sources))
-	for key := range s.cfg.Importer.Sources {
-		cfgEnabledSources[i] = key
-		i++
-	}
-	slices.Sort(cfgEnabledSources)
-
-	read, write, exists, err := index.OpenOrCreate(
-		s.cfg.DataPath,
-		options.Replace,
-		&index.Options{
-			LowMemory: options.LowMemory,
-			Logger:    options.Logger.Named("index"),
-		},
-	)
-	if err != nil {
-		return errors.Wrap(err, "Failed to open or create index")
-	}
-	s.readIndex = read
-	s.writeIndex = write
-
-	if !exists || options.Replace || options.Update {
-		s.log.Info(
-			"Starting build job",
-			"new",
-			!exists,
-			"replace",
-			options.Replace,
-			"update",
-			options.Update,
-		)
-		imp := importer.New(s.cfg, s.log.Named("importer"), write)
-		err = imp.Start(
-			ctx,
-			options.Replace || options.Update,
-			nil,
-		)
-		if err != nil {
-			return errors.Wrap(err, "Failed to build index")
-		}
-		if options.Replace || options.Update {
-			return nil
-		}
-	} else {
-		indexedSources, err := read.GetEnabledSources()
-		if err != nil {
-			return errors.Wrap(err, "Failed to get enabled sources from index")
-		}
-		slices.Sort(indexedSources)
-		if !slices.Equal(cfgEnabledSources, indexedSources) {
-			newSources := slices.DeleteFunc(slices.Clone(cfgEnabledSources), func(s string) bool {
-				return slices.Contains(indexedSources, s)
-			})
-			retiredSources := slices.DeleteFunc(slices.Clone(indexedSources), func(s string) bool {
-				return slices.Contains(cfgEnabledSources, s)
-			})
-			if len(newSources) > 0 {
-				s.log.Info("adding new sources", "sources", newSources)
-				imp := importer.New(s.cfg, options.Logger.Named("importer"), write)
-				err := imp.Start(
-					ctx,
-					false,
-					&newSources,
-				)
-				if err != nil {
-					return errors.Wrap(err, "Failed to update index with new sources")
-				}
-			}
-			if len(retiredSources) > 0 {
-				s.log.Info("removing retired sources", "sources", retiredSources)
-				for _, s := range retiredSources {
-					err := write.DeleteBySource(s)
-					if err != nil {
-						return errors.Wrapf(err, "Failed to remove retired source %s", s)
-					}
-				}
-			}
-		}
-	}
-	components.SetLastUpdated(read.LastUpdated())
-
-	return nil
-}
-
-type Server struct {
-	sv         *server.Server
-	wg         *sync.WaitGroup
-	cfg        *config.Config
-	log        *log.Logger
-	sentryHub  *sentry.Hub
-	readIndex  *index.ReadIndex
-	writeIndex *index.WriteIndex
-}
-
-func New(cfg *config.Config, log *log.Logger) (*Server, errors.E) {
-	err := sentry.Init(sentry.ClientOptions{
-		EnableTracing:    true,
-		TracesSampleRate: 1.0,
-		Dsn:              cfg.Web.SentryDSN,
-		Environment:      cfg.Web.Environment,
-	})
-	if err != nil {
-		log.Warn("could not initialise sentry", "error", err)
-	}
-
-	return &Server{
-		cfg:       cfg,
-		log:       log,
-		sentryHub: sentry.CurrentHub(),
-	}, nil
-}
-
-func (s *Server) startUpdateTimer(
-	ctx context.Context,
-	cfg *config.Config,
-	localHub *sentry.Hub,
-) {
-	const monitorSlug = "import"
-	localHub.WithScope(func(scope *sentry.Scope) {
-		var err errors.E
-		scope.SetContext("monitor", sentry.Context{"slug": monitorSlug})
-		monitorConfig := &sentry.MonitorConfig{
-			Schedule:      sentry.IntervalSchedule(1, sentry.MonitorScheduleUnitDay),
-			MaxRuntime:    int64(math.Ceil(cfg.Importer.Timeout.Minutes())),
-			CheckInMargin: 5,
-			Timezone:      time.Local.String(),
-		}
-
-		s.wg.Add(1)
-		nextRun := nextUTCOccurrenceOfTime(s.cfg.Importer.UpdateAt)
-		components.SetNextRun(nextRun)
-		for {
-			s.log.Debug("scheduling next run", "next-run", nextRun)
-			select {
-			case <-ctx.Done():
-				s.log.Debug("stopping scheduler")
-				s.wg.Done()
-
-				return
-			case <-time.After(time.Until(nextRun)):
-			}
-			s.wg.Add(1)
-			s.log.Info("updating index")
-
-			eventID := localHub.CaptureCheckIn(&sentry.CheckIn{
-				MonitorSlug: monitorSlug,
-				Status:      sentry.CheckInStatusInProgress,
-			}, monitorConfig)
-			components.MarkIndexingStarted()
-
-			imp := importer.New(s.cfg, s.log.Named("importer"), s.writeIndex)
-			err = imp.Start(ctx, false, nil)
-			s.wg.Done()
-			if err != nil {
-				s.log.Warn("error updating index", "error", err)
-
-				localHub.CaptureException(err)
-				localHub.CaptureCheckIn(&sentry.CheckIn{
-					ID:          *eventID,
-					MonitorSlug: monitorSlug,
-					Status:      sentry.CheckInStatusError,
-				}, monitorConfig)
-			} else {
-				s.log.Info("update complete")
-
-				localHub.CaptureCheckIn(&sentry.CheckIn{
-					ID:          *eventID,
-					MonitorSlug: monitorSlug,
-					Status:      sentry.CheckInStatusOK,
-				}, monitorConfig)
-			}
-			nextRun = nextRun.AddDate(0, 0, 1)
-			components.MarkIndexingFinished(nextRun)
-		}
-	})
-}
-
-func (s *Server) Start(ctx context.Context, liveReload bool) errors.E {
-	var err errors.E
-	s.sv, err = server.New(s.cfg, s.readIndex, s.log.Named("server"), liveReload)
-	if err != nil {
-		return errors.Wrap(err, "error setting up server")
-	}
-
-	s.wg = &sync.WaitGroup{}
-	go s.startUpdateTimer(ctx, s.cfg, sentry.CurrentHub().Clone())
-
-	s.wg.Add(1)
-	err = s.sv.Start()
-	if err != nil {
-		s.wg.Done()
-
-		return errors.Wrap(err, "error starting server")
-	}
-
-	return nil
-}
-
-func (s *Server) Stop() {
-	<-s.sv.Stop()
-	defer s.wg.Done()
-	s.sentryHub.Flush(2 * time.Second)
-}
diff --git a/web/searchix.go b/web/searchix.go
new file mode 100644
index 0000000..3197285
--- /dev/null
+++ b/web/searchix.go
@@ -0,0 +1,60 @@
+package web
+
+import (
+	"time"
+
+	"go.alanpearce.eu/searchix/internal/config"
+	"go.alanpearce.eu/searchix/internal/index"
+	"go.alanpearce.eu/searchix/internal/server"
+	"go.alanpearce.eu/x/log"
+
+	"github.com/getsentry/sentry-go"
+	"gitlab.com/tozd/go/errors"
+)
+
+type Server struct {
+	sv        *server.Server
+	cfg       *config.Config
+	log       *log.Logger
+	sentryHub *sentry.Hub
+	readIndex *index.ReadIndex
+}
+
+func New(cfg *config.Config, log *log.Logger, read *index.ReadIndex) (*Server, errors.E) {
+	err := sentry.Init(sentry.ClientOptions{
+		EnableTracing:    true,
+		TracesSampleRate: 1.0,
+		Dsn:              cfg.Web.SentryDSN,
+		Environment:      cfg.Web.Environment,
+	})
+	if err != nil {
+		log.Warn("could not initialise sentry", "error", err)
+	}
+
+	return &Server{
+		cfg:       cfg,
+		log:       log,
+		sentryHub: sentry.CurrentHub(),
+		readIndex: read,
+	}, nil
+}
+
+func (s *Server) Start(liveReload bool) errors.E {
+	var err errors.E
+	s.sv, err = server.New(s.cfg, s.readIndex, s.log.Named("server"), liveReload)
+	if err != nil {
+		return errors.Wrap(err, "error setting up server")
+	}
+
+	err = s.sv.Start()
+	if err != nil {
+		return errors.Wrap(err, "error starting server")
+	}
+
+	return nil
+}
+
+func (s *Server) Stop() {
+	<-s.sv.Stop()
+	s.sentryHub.Flush(2 * time.Second)
+}
generated by cgit-pink 1.4.1 (git 2.36.1) at 2025-03-26 01:10:01 +0000