perf: flush index batch in groups of 10000
Alan Pearce alan@alanpearce.eu
Fri, 17 May 2024 11:22:54 +0200
1 files changed, 26 insertions(+), 5 deletions(-)
jump to
M internal/index/indexer.go → internal/index/indexer.go
@@ -31,6 +31,8 @@ index bleve.Index meta *Meta } +const batchSize = 10_000 + func createIndexMapping() (mapping.IndexMapping, error) { indexMapping := bleve.NewIndexMapping() indexMapping.StoreDynamic = false @@ -233,6 +235,7 @@ errs := make(chan error) go func() { defer close(errs) + k := 0 batch := i.index.NewBatch() indexMapping := i.index.Mapping() @@ -273,18 +276,36 @@ errs <- errors.WithMessagef(err, "could not index object %s", obj.GetName()) continue } - } - size := batch.Size() - slog.Debug("flushing batch", "size", size) + if k++; k%batchSize == 0 { + err = i.Flush(batch) + if err != nil { + errs <- err + } + } + } - err := i.index.Batch(batch) + err := i.Flush(batch) if err != nil { - errs <- errors.WithMessagef(err, "could not flush batch") + errs <- err } }() return errs +} + +func (i *WriteIndex) Flush(batch *bleve.Batch) error { + size := batch.Size() + slog.Debug("flushing batch", "size", size) + + err := i.index.Batch(batch) + if err != nil { + return errors.WithMessagef(err, "could not flush batch") + } + + batch.Reset() + + return nil } func (i *WriteIndex) Close() error {