From 63599c719d8e4e09548546bb366a849ef40c602b Mon Sep 17 00:00:00 2001 From: Alan Pearce Date: Sun, 9 Jun 2024 17:54:57 +0200 Subject: feat: add low-memory mode --- internal/config/config.go | 2 +- internal/config/default.go | 9 +++++---- internal/config/structs.go | 7 ++++--- internal/importer/main_test.go | 26 ++++++++++++++++++++++++++ internal/index/indexer.go | 32 ++++++++++++++++++++++++-------- 5 files changed, 60 insertions(+), 16 deletions(-) create mode 100644 internal/importer/main_test.go (limited to 'internal') diff --git a/internal/config/config.go b/internal/config/config.go index c8739f0..83ddd2c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -91,7 +91,7 @@ func mustLocalTime(in string) (time LocalTime) { } func GetConfig(filename string) (*Config, error) { - config := defaultConfig + config := DefaultConfig if filename != "" { slog.Debug("reading config", "filename", filename) f, err := os.Open(filename) diff --git a/internal/config/default.go b/internal/config/default.go index 5e7b388..5b924a9 100644 --- a/internal/config/default.go +++ b/internal/config/default.go @@ -18,7 +18,7 @@ const self = "'self'" const maxAge = (1 * 365 * 24 * time.Hour) -var defaultConfig = Config{ +var DefaultConfig = Config{ DataPath: "./data", Web: &Web{ ListenAddress: "localhost", @@ -47,8 +47,9 @@ var defaultConfig = Config{ }, }, Importer: &Importer{ - Timeout: Duration{30 * time.Minute}, - UpdateAt: mustLocalTime("04:00:00"), + LowMemory: false, + Timeout: Duration{30 * time.Minute}, + UpdateAt: mustLocalTime("04:00:00"), Sources: map[string]*Source{ "nixos": { Name: "NixOS", @@ -116,7 +117,7 @@ var defaultConfig = Config{ } func GetDefaultConfig() string { - out, err := toml.Marshal(&defaultConfig) + out, err := toml.Marshal(&DefaultConfig) if err != nil { panic("could not read default configuration") } diff --git a/internal/config/structs.go b/internal/config/structs.go index a434698..70283f2 100644 --- a/internal/config/structs.go +++ b/internal/config/structs.go @@ -27,9 +27,10 @@ type Web struct { } type Importer struct { - Sources map[string]*Source - Timeout Duration `comment:"Abort fetch and import process for all jobs if it takes longer than this value."` - UpdateAt LocalTime `comment:"Local time of day to run fetch/import process"` + Sources map[string]*Source + LowMemory bool `comment:"Use less memory at the expense of import performance"` + Timeout Duration `comment:"Abort fetch and import process for all jobs if it takes longer than this value."` + UpdateAt LocalTime `comment:"Local time of day to run fetch/import process"` } type Source struct { diff --git a/internal/importer/main_test.go b/internal/importer/main_test.go new file mode 100644 index 0000000..bbb91a3 --- /dev/null +++ b/internal/importer/main_test.go @@ -0,0 +1,26 @@ +package importer + +import ( + "log/slog" + "searchix/internal/config" + "searchix/internal/index" + "testing" +) + +var cfg = config.DefaultConfig + +func BenchmarkImporterLowMemory(b *testing.B) { + tmp := b.TempDir() + cfg.LogLevel = slog.LevelDebug + _, write, _, err := index.OpenOrCreate(tmp, false, &index.Options{ + LowMemory: true, + }) + if err != nil { + b.Fatal(err) + } + + err = Start(&cfg, write, false, &[]string{"nixpkgs"}) + if err != nil { + b.Fatal(err) + } +} diff --git a/internal/index/indexer.go b/internal/index/indexer.go index a59889a..600ea9b 100644 --- a/internal/index/indexer.go +++ b/internal/index/indexer.go @@ -40,7 +40,7 @@ func (e *BatchError) Error() string { return e.error.Error() } -const batchSize = 10_000 +var batchSize = 10_000 func createIndexMapping() (mapping.IndexMapping, error) { indexMapping := bleve.NewIndexMapping() @@ -126,19 +126,24 @@ func createIndexMapping() (mapping.IndexMapping, error) { return indexMapping, nil } -func createIndex(indexPath string) (bleve.Index, error) { +func createIndex(indexPath string, options *Options) (bleve.Index, error) { indexMapping, err := createIndexMapping() if err != nil { return nil, err } + kvconfig := make(map[string]interface{}) + if options.LowMemory { + kvconfig = map[string]interface{}{ + "PersisterNapTimeMSec": 1000, + "PersisterNapUnderNumFiles": 500, + } + } idx, err := bleve.NewUsing( indexPath, indexMapping, bleve.Config.DefaultIndexType, bleve.Config.DefaultKVStore, - map[string]interface{}{ - "nosync": true, - }, + kvconfig, ) if err != nil { return nil, errors.WithMessagef(err, "unable to create index at path %s", indexPath) @@ -182,7 +187,15 @@ func deleteIndex(dataRoot string) error { return nil } -func OpenOrCreate(dataRoot string, force bool) (*ReadIndex, *WriteIndex, bool, error) { +type Options struct { + LowMemory bool +} + +func OpenOrCreate( + dataRoot string, + force bool, + options *Options, +) (*ReadIndex, *WriteIndex, bool, error) { var err error bleve.SetLog(log.Default()) @@ -207,7 +220,7 @@ func OpenOrCreate(dataRoot string, force bool) (*ReadIndex, *WriteIndex, bool, e return nil, nil, false, err } } - idx, err = createIndex(indexPath) + idx, err = createIndex(indexPath, options) if err != nil { return nil, nil, false, err } @@ -227,7 +240,10 @@ func OpenOrCreate(dataRoot string, force bool) (*ReadIndex, *WriteIndex, bool, e if err != nil { return nil, nil, exists, err } + } + if options.LowMemory { + batchSize = 1_000 } return &ReadIndex{ @@ -362,7 +378,7 @@ func (i *WriteIndex) DeleteBySource(source string) error { } batch := i.index.NewBatch() - var k uint + var k int for _, hit := range results.Hits { batch.Delete(hit.ID) if k++; k%batchSize == 0 { -- cgit 1.4.1