about summary refs log tree commit diff stats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.dir-locals.el4
-rw-r--r--.gitignore2
-rw-r--r--CHANGELOG.md20
-rw-r--r--cmd/searchix-web/main.go1
-rw-r--r--defaults.toml2
-rw-r--r--internal/components/packageDetail.go12
-rw-r--r--internal/config/default.go1
-rw-r--r--internal/config/structs.go1
-rw-r--r--internal/importer/main.go9
-rw-r--r--internal/importer/main_test.go1
-rw-r--r--internal/index/indexer.go30
-rw-r--r--internal/index/search_test.go1
-rw-r--r--nix/modules/default.nix2
-rw-r--r--nix/package.nix2
14 files changed, 64 insertions, 24 deletions
diff --git a/.dir-locals.el b/.dir-locals.el
deleted file mode 100644
index 02a101d..0000000
--- a/.dir-locals.el
+++ /dev/null
@@ -1,4 +0,0 @@
-;;; Directory Local Variables            -*- no-byte-compile: t -*-
-;;; For more information see (info "(emacs) Directory Variables")
-
-((go-ts-mode . ((apheleia-formatter . golines))))
diff --git a/.gitignore b/.gitignore
index be8e8fa..0f26772 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,8 +6,6 @@
 *.dll
 *.so
 *.dylib
-!*.*
-!/**/
 
 # Test binary, built with `go test -c`
 *.test
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ba1342f..9ccbf64 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,9 +1,27 @@
 # Changelog
 
-## [v0.1.33](https://git.alanpearce.eu/searchix/diff/?id=v0.1.32&id2=34bc16d3e3d12d4d14b9306baef8e5ecf600ebac) (2025-03-24)
+## [v0.1.34](https://git.alanpearce.eu/searchix/diff/?id=v0.1.33&id2=abf819db104560670cdc388a584d09968aac7bbb) (2025-03-24)
+
+### Features
+
+- buffer fetched HTTP bodies
+  ([abf819d](https://git.alanpearce.eu/searchix/commit/?id=abf819db104560670cdc388a584d09968aac7bbb))
+- re-index on startup if last run > 24 hours ago
+  ([6984d4d](https://git.alanpearce.eu/searchix/commit/?id=6984d4d32ab506494394f8a6a8aa18041c45b9e8))
 
 ### Fixes
 
+- another typo in search markup
+  ([efaaf3f](https://git.alanpearce.eu/searchix/commit/?id=efaaf3f5296807eec377b19d4733fd360c8c655f))
+- typo in indexing status
+  ([fecc84f](https://git.alanpearce.eu/searchix/commit/?id=fecc84fab93bb3228be5fee4160820751195feec))
+- nil pointer panic in importer
+  ([0fb48cf](https://git.alanpearce.eu/searchix/commit/?id=0fb48cf6dd934778584120ebbc9e568c8b80b80e))
+
+### [v0.1.33](https://git.alanpearce.eu/searchix/diff/?id=v0.1.32&id2=v0.1.33) (2025-03-24)
+
+#### Fixes
+
 - build failure due to outdated file reference
   ([dd58d99](https://git.alanpearce.eu/searchix/commit/?id=dd58d9925b68c7cd639c5e217782709c1936497f))
 
diff --git a/cmd/searchix-web/main.go b/cmd/searchix-web/main.go
index 20d4a94..061e08d 100644
--- a/cmd/searchix-web/main.go
+++ b/cmd/searchix-web/main.go
@@ -77,6 +77,7 @@ func main() {
 		*replace,
 		&index.Options{
 			LowMemory: cfg.Importer.LowMemory,
+			BatchSize: cfg.Importer.BatchSize,
 			Logger:    logger.Named("index"),
 		},
 	)
diff --git a/defaults.toml b/defaults.toml
index f75aae6..2ceb0eb 100644
--- a/defaults.toml
+++ b/defaults.toml
@@ -64,6 +64,8 @@ x-frame-options = 'DENY'
 [Importer]
 # Use less memory at the expense of import performance
 LowMemory = false
+# Number of items to process in each batch (affects memory usage).
+BatchSize = 10000
 # Abort fetch and import process for all jobs if it takes longer than this value.
 Timeout = '30m0s'
 # Time of day (UTC) to run fetch/import process
diff --git a/internal/components/packageDetail.go b/internal/components/packageDetail.go
index 01b1f4d..186fb60 100644
--- a/internal/components/packageDetail.go
+++ b/internal/components/packageDetail.go
@@ -105,6 +105,18 @@ func PackageDetail(pkg nix.Package) g.Node {
 					),
 				}),
 			),
+			g.If(len(pkg.Platforms) > 0,
+				g.Group([]g.Node{
+					Dt(g.Text("Platforms")),
+					Dd(
+						Ul(
+							g.Map(pkg.Platforms, func(plat string) g.Node {
+								return Li(g.Text(plat))
+							}),
+						),
+					),
+				}),
+			),
 			g.If(pkg.Definition != "",
 				g.Group([]g.Node{
 					Dt(g.Text("Defined")),
diff --git a/internal/config/default.go b/internal/config/default.go
index 5260fe9..0ec0735 100644
--- a/internal/config/default.go
+++ b/internal/config/default.go
@@ -49,6 +49,7 @@ var DefaultConfig = Config{
 	},
 	Importer: &Importer{
 		LowMemory: false,
+		BatchSize: 10_000,
 		Timeout:   Duration{30 * time.Minute},
 		UpdateAt:  mustLocalTime("03:00:00"),
 		Sources: map[string]*Source{
diff --git a/internal/config/structs.go b/internal/config/structs.go
index 30ad975..52d0f0e 100644
--- a/internal/config/structs.go
+++ b/internal/config/structs.go
@@ -31,6 +31,7 @@ type Web struct {
 type Importer struct {
 	Sources   map[string]*Source
 	LowMemory bool      `comment:"Use less memory at the expense of import performance"`
+	BatchSize int       `comment:"Number of items to process in each batch (affects memory usage)."`
 	Timeout   Duration  `comment:"Abort fetch and import process for all jobs if it takes longer than this value."`
 	UpdateAt  LocalTime `comment:"Time of day (UTC) to run fetch/import process"`
 }
diff --git a/internal/importer/main.go b/internal/importer/main.go
index 184c6df..d129cd3 100644
--- a/internal/importer/main.go
+++ b/internal/importer/main.go
@@ -286,7 +286,7 @@ func (imp *Importer) EnsureSourcesIndexed(
 }
 
 func (imp *Importer) StartUpdateTimer(
-	ctx context.Context,
+	parentCtx context.Context,
 	localHub *sentry.Hub,
 ) {
 	const monitorSlug = "import"
@@ -309,9 +309,8 @@ func (imp *Importer) StartUpdateTimer(
 		}
 		SetNextRun(nextRun)
 		for {
-			imp.log.Debug("scheduling next run", "next-run", nextRun)
 			select {
-			case <-ctx.Done():
+			case <-parentCtx.Done():
 				imp.log.Debug("stopping scheduler")
 
 				return
@@ -325,7 +324,10 @@ func (imp *Importer) StartUpdateTimer(
 			}, monitorConfig)
 			MarkIndexingStarted()
 
+			ctx, cancel := context.WithTimeout(parentCtx, imp.config.Importer.Timeout.Duration)
 			err = imp.Start(ctx, false, nil)
+			cancel()
+
 			if err != nil {
 				imp.log.Warn("error updating index", "error", err)
 
@@ -346,6 +348,7 @@ func (imp *Importer) StartUpdateTimer(
 			}
 			nextRun = nextUTCOccurrenceOfTime(imp.config.Importer.UpdateAt)
 			MarkIndexingFinished(nextRun)
+			imp.log.Info("scheduling next run", "next-run", nextRun.Format(time.DateTime))
 		}
 	})
 }
diff --git a/internal/importer/main_test.go b/internal/importer/main_test.go
index 84f6adf..eb155e0 100644
--- a/internal/importer/main_test.go
+++ b/internal/importer/main_test.go
@@ -16,6 +16,7 @@ func BenchmarkImporterLowMemory(b *testing.B) {
 	logger := log.Configure(false)
 	_, write, _, err := index.OpenOrCreate(tmp, false, &index.Options{
 		LowMemory: true,
+		BatchSize: cfg.Importer.BatchSize,
 		Logger:    logger.Named("index"),
 	})
 	if err != nil {
diff --git a/internal/index/indexer.go b/internal/index/indexer.go
index 7591aef..454a736 100644
--- a/internal/index/indexer.go
+++ b/internal/index/indexer.go
@@ -11,6 +11,7 @@ import (
 	"path/filepath"
 	"slices"
 
+	"go.alanpearce.eu/searchix/internal/config"
 	"go.alanpearce.eu/searchix/internal/file"
 	"go.alanpearce.eu/searchix/internal/nix"
 	"go.alanpearce.eu/x/log"
@@ -34,13 +35,15 @@ import (
 
 type Options struct {
 	LowMemory bool
+	BatchSize int
 	Logger    *log.Logger
 }
 
 type WriteIndex struct {
-	index bleve.Index
-	log   *log.Logger
-	Meta  *Meta
+	batchSize int
+	index     bleve.Index
+	log       *log.Logger
+	Meta      *Meta
 }
 
 type BatchError struct {
@@ -51,8 +54,6 @@ func (e *BatchError) Error() string {
 	return e.E.Error()
 }
 
-var batchSize = 10_000
-
 func createIndexMapping() (mapping.IndexMapping, errors.E) {
 	indexMapping := bleve.NewIndexMapping()
 	indexMapping.StoreDynamic = false
@@ -268,8 +269,12 @@ func OpenOrCreate(
 		}
 	}
 
-	if options.LowMemory {
-		batchSize = 1_000
+	if options.BatchSize == 0 {
+		options.BatchSize = config.DefaultConfig.Importer.BatchSize
+	}
+
+	if options.LowMemory && options.BatchSize == config.DefaultConfig.Importer.BatchSize {
+		options.BatchSize = 1_000
 	}
 
 	return &ReadIndex{
@@ -278,9 +283,10 @@ func OpenOrCreate(
 			meta:  meta,
 		},
 		&WriteIndex{
-			index: idx,
-			log:   options.Logger,
-			Meta:  meta,
+			index:     idx,
+			batchSize: options.BatchSize,
+			log:       options.Logger,
+			Meta:      meta,
 		},
 		exists,
 		nil
@@ -337,7 +343,7 @@ func (i *WriteIndex) Import(
 				continue
 			}
 
-			if k++; k%batchSize == 0 {
+			if k++; k%i.batchSize == 0 {
 				err = i.Flush(batch)
 				if err != nil {
 					errs <- err
@@ -405,7 +411,7 @@ func (i *WriteIndex) DeleteBySource(source string) errors.E {
 	var k int
 	for _, hit := range results.Hits {
 		batch.Delete(hit.ID)
-		if k++; k%batchSize == 0 {
+		if k++; k%i.batchSize == 0 {
 			err := i.Flush(batch)
 			if err != nil {
 				return err
diff --git a/internal/index/search_test.go b/internal/index/search_test.go
index 339a0de..126c0a6 100644
--- a/internal/index/search_test.go
+++ b/internal/index/search_test.go
@@ -22,6 +22,7 @@ func TestSearchGitPackagesFirst(t *testing.T) {
 
 	read, _, exists, err := index.OpenOrCreate(dataRoot, false, &index.Options{
 		Logger:    log.Named("index"),
+		BatchSize: cfg.Importer.BatchSize,
 		LowMemory: false,
 	})
 	defer read.Close()
diff --git a/nix/modules/default.nix b/nix/modules/default.nix
index e0c8bf3..3aba4da 100644
--- a/nix/modules/default.nix
+++ b/nix/modules/default.nix
@@ -200,7 +200,7 @@ in
       description = ''
         Configuration for searchix.
 
-        See https://git.alanpearce.eu/searchix/tree/defaults.toml?h=v0.1.33
+        See https://git.alanpearce.eu/searchix/tree/defaults.toml?h=v0.1.34
       '';
     };
   };
diff --git a/nix/package.nix b/nix/package.nix
index 8c81386..c2c1054 100644
--- a/nix/package.nix
+++ b/nix/package.nix
@@ -13,7 +13,7 @@
 , css
 }:
 let
-  version = "0.1.33";
+  version = "0.1.34";
 in
 buildGoApplication {
   pname = "searchix";
generated by cgit-pink 1.4.1 (git 2.36.1) at 2025-03-28 20:22:41 +0000