1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
|
package importer
import (
"context"
"sync"
"go.alanpearce.eu/searchix/internal/index"
"go.alanpearce.eu/searchix/internal/nix"
"go.alanpearce.eu/x/log"
)
type Importer interface {
Import(context.Context, *index.WriteIndex) (bool, error)
}
type Processor interface {
Process(context.Context) (<-chan nix.Importable, <-chan error)
}
func process(
ctx context.Context,
indexer *index.WriteIndex,
processor Processor,
logger *log.Logger,
) (bool, error) {
wg := sync.WaitGroup{}
wg.Add(1)
objects, pErrs := processor.Process(ctx)
wg.Add(1)
iErrs := indexer.Import(ctx, objects)
var hadObjectErrors bool
var criticalError error
go func() {
for {
select {
case err, running := <-iErrs:
if !running {
wg.Done()
iErrs = nil
logger.Debug("ingest completed")
continue
}
be, isBatchError := err.(*index.BatchError)
if isBatchError {
criticalError = be
break
}
hadObjectErrors = true
logger.Warn("error ingesting object", "error", err)
case err, running := <-pErrs:
if !running {
wg.Done()
pErrs = nil
logger.Debug("processing completed")
continue
}
hadObjectErrors = true
logger.Warn("error processing object", "error", err)
}
}
}()
wg.Wait()
return hadObjectErrors, criticalError
}
|