about summary refs log tree commit diff stats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.dir-locals.el4
-rw-r--r--.gitignore2
-rw-r--r--cmd/searchix-web/main.go1
-rw-r--r--defaults.toml2
-rw-r--r--frontend/static/search.js12
-rw-r--r--internal/components/packageDetail.go12
-rw-r--r--internal/config/default.go1
-rw-r--r--internal/config/structs.go1
-rw-r--r--internal/importer/main.go9
-rw-r--r--internal/importer/main_test.go1
-rw-r--r--internal/index/indexer.go30
-rw-r--r--internal/index/search_test.go1
12 files changed, 44 insertions, 32 deletions
diff --git a/.dir-locals.el b/.dir-locals.el
deleted file mode 100644
index 02a101d..0000000
--- a/.dir-locals.el
+++ /dev/null
@@ -1,4 +0,0 @@
-;;; Directory Local Variables            -*- no-byte-compile: t -*-
-;;; For more information see (info "(emacs) Directory Variables")
-
-((go-ts-mode . ((apheleia-formatter . golines))))
diff --git a/.gitignore b/.gitignore
index be8e8fa..0f26772 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,8 +6,6 @@
 *.dll
 *.so
 *.dylib
-!*.*
-!/**/
 
 # Test binary, built with `go test -c`
 *.test
diff --git a/cmd/searchix-web/main.go b/cmd/searchix-web/main.go
index 20d4a94..061e08d 100644
--- a/cmd/searchix-web/main.go
+++ b/cmd/searchix-web/main.go
@@ -77,6 +77,7 @@ func main() {
 		*replace,
 		&index.Options{
 			LowMemory: cfg.Importer.LowMemory,
+			BatchSize: cfg.Importer.BatchSize,
 			Logger:    logger.Named("index"),
 		},
 	)
diff --git a/defaults.toml b/defaults.toml
index f75aae6..2ceb0eb 100644
--- a/defaults.toml
+++ b/defaults.toml
@@ -64,6 +64,8 @@ x-frame-options = 'DENY'
 [Importer]
 # Use less memory at the expense of import performance
 LowMemory = false
+# Number of items to process in each batch (affects memory usage).
+BatchSize = 10000
 # Abort fetch and import process for all jobs if it takes longer than this value.
 Timeout = '30m0s'
 # Time of day (UTC) to run fetch/import process
diff --git a/frontend/static/search.js b/frontend/static/search.js
index 0bf4a5e..41f40bc 100644
--- a/frontend/static/search.js
+++ b/frontend/static/search.js
@@ -18,7 +18,6 @@ let state = history.state || {
   url: urlLocation.toString(),
   input: urlLocation.searchParams.get("query"),
   results: resultsRange.cloneContents().innerHTML || null,
-  opened: [],
 };
 
 let escapePolicy = null;
@@ -105,7 +104,6 @@ async function getResults(url) {
     // render errors sent as HTML as well as OK responses
     if (res.headers.get("content-type").startsWith("text/html")) {
       state.results = await res.text();
-      state.opened = [];
       history.replaceState(state, null, url);
       renderResults(state.results);
     } else {
@@ -155,7 +153,7 @@ document.querySelector("a.current").addEventListener("click", function (ev) {
   state.results = "";
   history.pushState(state, null, ev.target.href);
   ev.preventDefault();
-  document.querySelectorAll("nav > a:not(.current)").forEach(function (node) {
+  nav.querySelectorAll("a:not(.current)").forEach(function (node) {
     const u = new URL(node.href);
     u.search = "";
     node.href = u.toString();
@@ -218,14 +216,6 @@ function handleDialogOpen(ev) {
   }
 }
 
-if (state.opened.length > 0) {
-  state.opened.forEach((id) =>
-    document.getElementById(id).setAttribute("open", "open"),
-  );
-} else if (location.hash) {
-  document.getElementById(location.hash.slice(1)).setAttribute("open", "open");
-}
-
 addEventListener("popstate", function (ev) {
   if (ev.state != null) {
     url = new URL(ev.state.url);
diff --git a/internal/components/packageDetail.go b/internal/components/packageDetail.go
index 01b1f4d..186fb60 100644
--- a/internal/components/packageDetail.go
+++ b/internal/components/packageDetail.go
@@ -105,6 +105,18 @@ func PackageDetail(pkg nix.Package) g.Node {
 					),
 				}),
 			),
+			g.If(len(pkg.Platforms) > 0,
+				g.Group([]g.Node{
+					Dt(g.Text("Platforms")),
+					Dd(
+						Ul(
+							g.Map(pkg.Platforms, func(plat string) g.Node {
+								return Li(g.Text(plat))
+							}),
+						),
+					),
+				}),
+			),
 			g.If(pkg.Definition != "",
 				g.Group([]g.Node{
 					Dt(g.Text("Defined")),
diff --git a/internal/config/default.go b/internal/config/default.go
index 5260fe9..0ec0735 100644
--- a/internal/config/default.go
+++ b/internal/config/default.go
@@ -49,6 +49,7 @@ var DefaultConfig = Config{
 	},
 	Importer: &Importer{
 		LowMemory: false,
+		BatchSize: 10_000,
 		Timeout:   Duration{30 * time.Minute},
 		UpdateAt:  mustLocalTime("03:00:00"),
 		Sources: map[string]*Source{
diff --git a/internal/config/structs.go b/internal/config/structs.go
index 30ad975..52d0f0e 100644
--- a/internal/config/structs.go
+++ b/internal/config/structs.go
@@ -31,6 +31,7 @@ type Web struct {
 type Importer struct {
 	Sources   map[string]*Source
 	LowMemory bool      `comment:"Use less memory at the expense of import performance"`
+	BatchSize int       `comment:"Number of items to process in each batch (affects memory usage)."`
 	Timeout   Duration  `comment:"Abort fetch and import process for all jobs if it takes longer than this value."`
 	UpdateAt  LocalTime `comment:"Time of day (UTC) to run fetch/import process"`
 }
diff --git a/internal/importer/main.go b/internal/importer/main.go
index 184c6df..d129cd3 100644
--- a/internal/importer/main.go
+++ b/internal/importer/main.go
@@ -286,7 +286,7 @@ func (imp *Importer) EnsureSourcesIndexed(
 }
 
 func (imp *Importer) StartUpdateTimer(
-	ctx context.Context,
+	parentCtx context.Context,
 	localHub *sentry.Hub,
 ) {
 	const monitorSlug = "import"
@@ -309,9 +309,8 @@ func (imp *Importer) StartUpdateTimer(
 		}
 		SetNextRun(nextRun)
 		for {
-			imp.log.Debug("scheduling next run", "next-run", nextRun)
 			select {
-			case <-ctx.Done():
+			case <-parentCtx.Done():
 				imp.log.Debug("stopping scheduler")
 
 				return
@@ -325,7 +324,10 @@ func (imp *Importer) StartUpdateTimer(
 			}, monitorConfig)
 			MarkIndexingStarted()
 
+			ctx, cancel := context.WithTimeout(parentCtx, imp.config.Importer.Timeout.Duration)
 			err = imp.Start(ctx, false, nil)
+			cancel()
+
 			if err != nil {
 				imp.log.Warn("error updating index", "error", err)
 
@@ -346,6 +348,7 @@ func (imp *Importer) StartUpdateTimer(
 			}
 			nextRun = nextUTCOccurrenceOfTime(imp.config.Importer.UpdateAt)
 			MarkIndexingFinished(nextRun)
+			imp.log.Info("scheduling next run", "next-run", nextRun.Format(time.DateTime))
 		}
 	})
 }
diff --git a/internal/importer/main_test.go b/internal/importer/main_test.go
index 84f6adf..eb155e0 100644
--- a/internal/importer/main_test.go
+++ b/internal/importer/main_test.go
@@ -16,6 +16,7 @@ func BenchmarkImporterLowMemory(b *testing.B) {
 	logger := log.Configure(false)
 	_, write, _, err := index.OpenOrCreate(tmp, false, &index.Options{
 		LowMemory: true,
+		BatchSize: cfg.Importer.BatchSize,
 		Logger:    logger.Named("index"),
 	})
 	if err != nil {
diff --git a/internal/index/indexer.go b/internal/index/indexer.go
index 7591aef..454a736 100644
--- a/internal/index/indexer.go
+++ b/internal/index/indexer.go
@@ -11,6 +11,7 @@ import (
 	"path/filepath"
 	"slices"
 
+	"go.alanpearce.eu/searchix/internal/config"
 	"go.alanpearce.eu/searchix/internal/file"
 	"go.alanpearce.eu/searchix/internal/nix"
 	"go.alanpearce.eu/x/log"
@@ -34,13 +35,15 @@ import (
 
 type Options struct {
 	LowMemory bool
+	BatchSize int
 	Logger    *log.Logger
 }
 
 type WriteIndex struct {
-	index bleve.Index
-	log   *log.Logger
-	Meta  *Meta
+	batchSize int
+	index     bleve.Index
+	log       *log.Logger
+	Meta      *Meta
 }
 
 type BatchError struct {
@@ -51,8 +54,6 @@ func (e *BatchError) Error() string {
 	return e.E.Error()
 }
 
-var batchSize = 10_000
-
 func createIndexMapping() (mapping.IndexMapping, errors.E) {
 	indexMapping := bleve.NewIndexMapping()
 	indexMapping.StoreDynamic = false
@@ -268,8 +269,12 @@ func OpenOrCreate(
 		}
 	}
 
-	if options.LowMemory {
-		batchSize = 1_000
+	if options.BatchSize == 0 {
+		options.BatchSize = config.DefaultConfig.Importer.BatchSize
+	}
+
+	if options.LowMemory && options.BatchSize == config.DefaultConfig.Importer.BatchSize {
+		options.BatchSize = 1_000
 	}
 
 	return &ReadIndex{
@@ -278,9 +283,10 @@ func OpenOrCreate(
 			meta:  meta,
 		},
 		&WriteIndex{
-			index: idx,
-			log:   options.Logger,
-			Meta:  meta,
+			index:     idx,
+			batchSize: options.BatchSize,
+			log:       options.Logger,
+			Meta:      meta,
 		},
 		exists,
 		nil
@@ -337,7 +343,7 @@ func (i *WriteIndex) Import(
 				continue
 			}
 
-			if k++; k%batchSize == 0 {
+			if k++; k%i.batchSize == 0 {
 				err = i.Flush(batch)
 				if err != nil {
 					errs <- err
@@ -405,7 +411,7 @@ func (i *WriteIndex) DeleteBySource(source string) errors.E {
 	var k int
 	for _, hit := range results.Hits {
 		batch.Delete(hit.ID)
-		if k++; k%batchSize == 0 {
+		if k++; k%i.batchSize == 0 {
 			err := i.Flush(batch)
 			if err != nil {
 				return err
diff --git a/internal/index/search_test.go b/internal/index/search_test.go
index 339a0de..126c0a6 100644
--- a/internal/index/search_test.go
+++ b/internal/index/search_test.go
@@ -22,6 +22,7 @@ func TestSearchGitPackagesFirst(t *testing.T) {
 
 	read, _, exists, err := index.OpenOrCreate(dataRoot, false, &index.Options{
 		Logger:    log.Named("index"),
+		BatchSize: cfg.Importer.BatchSize,
 		LowMemory: false,
 	})
 	defer read.Close()
generated by cgit-pink 1.4.1 (git 2.36.1) at 2025-03-30 15:04:11 +0000