1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
|
package importer
import (
"context"
"log/slog"
"path"
"searchix/internal/search"
"sync"
"time"
)
type Source struct {
Name string
Enable bool
Type Type
Channel string
Attribute string
ImportPath string `toml:"import-path"`
FetchTimeout time.Duration `toml:"fetch-timeout"`
ImportTimeout time.Duration `toml:"import-timeout"`
OutputPath string `toml:"output-path"`
Repo Repository
}
type Importer interface {
FetchIfNeeded(context.Context) (bool, error)
Import(context.Context, *search.WriteIndex) (bool, error)
}
func NewNixpkgsChannelImporter(
source Source,
dataPath string,
logger *slog.Logger,
) *NixpkgsChannelImporter {
indexPath := dataPath
fullpath := path.Join(dataPath, source.Channel)
return &NixpkgsChannelImporter{
DataPath: fullpath,
Source: source,
Logger: logger,
indexPath: indexPath,
}
}
func NewChannelImporter(source Source, dataPath string, logger *slog.Logger) *ChannelImporter {
indexPath := dataPath
fullpath := path.Join(dataPath, source.Channel)
return &ChannelImporter{
DataPath: fullpath,
Source: source,
Logger: logger,
indexPath: indexPath,
}
}
type importConfig struct {
IndexPath string
Filename string
Source Source
Logger *slog.Logger
}
func processOptions(
parent context.Context,
indexer *search.WriteIndex,
conf *importConfig,
) (bool, error) {
ctx, cancel := context.WithTimeout(parent, conf.Source.ImportTimeout)
defer cancel()
conf.Logger.Debug("creating option processor", "filename", conf.Filename)
processor, err := NewOptionProcessor(conf.Filename, conf.Source)
if err != nil {
return true, err
}
wg := sync.WaitGroup{}
wg.Add(1)
options, pErrs := processor.Process(ctx)
wg.Add(1)
iErrs := indexer.ImportOptions(ctx, options)
var hadErrors bool
go func() {
for {
select {
case err, running := <-iErrs:
if !running {
wg.Done()
iErrs = nil
slog.Info("ingest completed")
continue
}
hadErrors = true
conf.Logger.Warn("error ingesting option", "error", err)
case err, running := <-pErrs:
if !running {
wg.Done()
pErrs = nil
slog.Info("processing completed")
continue
}
hadErrors = true
conf.Logger.Warn("error processing option", "error", err)
}
}
}()
slog.Debug("options processing", "state", "waiting")
wg.Wait()
slog.Debug("options processing", "state", "complete")
return hadErrors, nil
}
|