1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
|
package importer
import (
"context"
"log/slog"
"searchix/internal/config"
"searchix/internal/index"
"sync"
)
type Importer interface {
FetchIfNeeded(context.Context) (bool, error)
Import(context.Context, *index.WriteIndex) (bool, error)
}
func NewNixpkgsChannelImporter(
source *config.Source,
dataPath string,
logger *slog.Logger,
) *NixpkgsChannelImporter {
return &NixpkgsChannelImporter{
DataPath: dataPath,
Source: source,
Logger: logger,
}
}
func NewChannelImporter(
source *config.Source,
dataPath string,
logger *slog.Logger,
) *ChannelImporter {
return &ChannelImporter{
DataPath: dataPath,
Source: source,
Logger: logger,
}
}
func NewDownloadOptionsImporter(
source *config.Source,
dataPath string,
logger *slog.Logger,
) *DownloadOptionsImporter {
return &DownloadOptionsImporter{
DataPath: dataPath,
Source: source,
Logger: logger,
}
}
type importConfig struct {
Filename string
Source *config.Source
Logger *slog.Logger
}
func processOptions(
parent context.Context,
indexer *index.WriteIndex,
conf *importConfig,
) (bool, error) {
ctx, cancel := context.WithTimeout(parent, conf.Source.ImportTimeout)
defer cancel()
conf.Logger.Debug("creating option processor", "filename", conf.Filename)
processor, err := NewOptionProcessor(conf.Filename, conf.Source)
if err != nil {
return true, err
}
wg := sync.WaitGroup{}
wg.Add(1)
options, pErrs := processor.Process(ctx)
wg.Add(1)
iErrs := indexer.Import(ctx, options)
var hadErrors bool
go func() {
for {
select {
case err, running := <-iErrs:
if !running {
wg.Done()
iErrs = nil
conf.Logger.Info("ingest completed")
continue
}
hadErrors = true
conf.Logger.Warn("error ingesting option", "error", err)
case err, running := <-pErrs:
if !running {
wg.Done()
pErrs = nil
conf.Logger.Debug("processing completed")
continue
}
hadErrors = true
conf.Logger.Warn("error processing option", "error", err)
}
}
}()
conf.Logger.Debug("options processing", "state", "waiting")
wg.Wait()
conf.Logger.Debug("options processing", "state", "complete")
return hadErrors, nil
}
|