diff options
Diffstat (limited to 'internal')
| -rw-r--r-- | internal/components/packageDetail.go | 12 | ||||
| -rw-r--r-- | internal/config/default.go | 1 | ||||
| -rw-r--r-- | internal/config/structs.go | 1 | ||||
| -rw-r--r-- | internal/importer/main.go | 9 | ||||
| -rw-r--r-- | internal/importer/main_test.go | 1 | ||||
| -rw-r--r-- | internal/index/indexer.go | 30 | ||||
| -rw-r--r-- | internal/index/search_test.go | 1 |
7 files changed, 40 insertions, 15 deletions
diff --git a/internal/components/packageDetail.go b/internal/components/packageDetail.go index 01b1f4d..186fb60 100644 --- a/internal/components/packageDetail.go +++ b/internal/components/packageDetail.go @@ -105,6 +105,18 @@ func PackageDetail(pkg nix.Package) g.Node { ), }), ), + g.If(len(pkg.Platforms) > 0, + g.Group([]g.Node{ + Dt(g.Text("Platforms")), + Dd( + Ul( + g.Map(pkg.Platforms, func(plat string) g.Node { + return Li(g.Text(plat)) + }), + ), + ), + }), + ), g.If(pkg.Definition != "", g.Group([]g.Node{ Dt(g.Text("Defined")), diff --git a/internal/config/default.go b/internal/config/default.go index 5260fe9..0ec0735 100644 --- a/internal/config/default.go +++ b/internal/config/default.go @@ -49,6 +49,7 @@ var DefaultConfig = Config{ }, Importer: &Importer{ LowMemory: false, + BatchSize: 10_000, Timeout: Duration{30 * time.Minute}, UpdateAt: mustLocalTime("03:00:00"), Sources: map[string]*Source{ diff --git a/internal/config/structs.go b/internal/config/structs.go index 30ad975..52d0f0e 100644 --- a/internal/config/structs.go +++ b/internal/config/structs.go @@ -31,6 +31,7 @@ type Web struct { type Importer struct { Sources map[string]*Source LowMemory bool `comment:"Use less memory at the expense of import performance"` + BatchSize int `comment:"Number of items to process in each batch (affects memory usage)."` Timeout Duration `comment:"Abort fetch and import process for all jobs if it takes longer than this value."` UpdateAt LocalTime `comment:"Time of day (UTC) to run fetch/import process"` } diff --git a/internal/importer/main.go b/internal/importer/main.go index 184c6df..d129cd3 100644 --- a/internal/importer/main.go +++ b/internal/importer/main.go @@ -286,7 +286,7 @@ func (imp *Importer) EnsureSourcesIndexed( } func (imp *Importer) StartUpdateTimer( - ctx context.Context, + parentCtx context.Context, localHub *sentry.Hub, ) { const monitorSlug = "import" @@ -309,9 +309,8 @@ func (imp *Importer) StartUpdateTimer( } SetNextRun(nextRun) for { - imp.log.Debug("scheduling next run", "next-run", nextRun) select { - case <-ctx.Done(): + case <-parentCtx.Done(): imp.log.Debug("stopping scheduler") return @@ -325,7 +324,10 @@ func (imp *Importer) StartUpdateTimer( }, monitorConfig) MarkIndexingStarted() + ctx, cancel := context.WithTimeout(parentCtx, imp.config.Importer.Timeout.Duration) err = imp.Start(ctx, false, nil) + cancel() + if err != nil { imp.log.Warn("error updating index", "error", err) @@ -346,6 +348,7 @@ func (imp *Importer) StartUpdateTimer( } nextRun = nextUTCOccurrenceOfTime(imp.config.Importer.UpdateAt) MarkIndexingFinished(nextRun) + imp.log.Info("scheduling next run", "next-run", nextRun.Format(time.DateTime)) } }) } diff --git a/internal/importer/main_test.go b/internal/importer/main_test.go index 84f6adf..eb155e0 100644 --- a/internal/importer/main_test.go +++ b/internal/importer/main_test.go @@ -16,6 +16,7 @@ func BenchmarkImporterLowMemory(b *testing.B) { logger := log.Configure(false) _, write, _, err := index.OpenOrCreate(tmp, false, &index.Options{ LowMemory: true, + BatchSize: cfg.Importer.BatchSize, Logger: logger.Named("index"), }) if err != nil { diff --git a/internal/index/indexer.go b/internal/index/indexer.go index 7591aef..454a736 100644 --- a/internal/index/indexer.go +++ b/internal/index/indexer.go @@ -11,6 +11,7 @@ import ( "path/filepath" "slices" + "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/file" "go.alanpearce.eu/searchix/internal/nix" "go.alanpearce.eu/x/log" @@ -34,13 +35,15 @@ import ( type Options struct { LowMemory bool + BatchSize int Logger *log.Logger } type WriteIndex struct { - index bleve.Index - log *log.Logger - Meta *Meta + batchSize int + index bleve.Index + log *log.Logger + Meta *Meta } type BatchError struct { @@ -51,8 +54,6 @@ func (e *BatchError) Error() string { return e.E.Error() } -var batchSize = 10_000 - func createIndexMapping() (mapping.IndexMapping, errors.E) { indexMapping := bleve.NewIndexMapping() indexMapping.StoreDynamic = false @@ -268,8 +269,12 @@ func OpenOrCreate( } } - if options.LowMemory { - batchSize = 1_000 + if options.BatchSize == 0 { + options.BatchSize = config.DefaultConfig.Importer.BatchSize + } + + if options.LowMemory && options.BatchSize == config.DefaultConfig.Importer.BatchSize { + options.BatchSize = 1_000 } return &ReadIndex{ @@ -278,9 +283,10 @@ func OpenOrCreate( meta: meta, }, &WriteIndex{ - index: idx, - log: options.Logger, - Meta: meta, + index: idx, + batchSize: options.BatchSize, + log: options.Logger, + Meta: meta, }, exists, nil @@ -337,7 +343,7 @@ func (i *WriteIndex) Import( continue } - if k++; k%batchSize == 0 { + if k++; k%i.batchSize == 0 { err = i.Flush(batch) if err != nil { errs <- err @@ -405,7 +411,7 @@ func (i *WriteIndex) DeleteBySource(source string) errors.E { var k int for _, hit := range results.Hits { batch.Delete(hit.ID) - if k++; k%batchSize == 0 { + if k++; k%i.batchSize == 0 { err := i.Flush(batch) if err != nil { return err diff --git a/internal/index/search_test.go b/internal/index/search_test.go index 339a0de..126c0a6 100644 --- a/internal/index/search_test.go +++ b/internal/index/search_test.go @@ -22,6 +22,7 @@ func TestSearchGitPackagesFirst(t *testing.T) { read, _, exists, err := index.OpenOrCreate(dataRoot, false, &index.Options{ Logger: log.Named("index"), + BatchSize: cfg.Importer.BatchSize, LowMemory: false, }) defer read.Close() |