diff options
| -rw-r--r-- | internal/config/config.go | 19 | ||||
| -rw-r--r-- | internal/config/fetcher.go | 49 | ||||
| -rw-r--r-- | internal/config/importer-type.go | 44 | ||||
| -rw-r--r-- | internal/config/repository.go | 5 | ||||
| -rw-r--r-- | internal/config/source.go | 48 | ||||
| -rw-r--r-- | internal/fetcher/channel.go (renamed from internal/importer/channel.go) | 50 | ||||
| -rw-r--r-- | internal/fetcher/download.go | 75 | ||||
| -rw-r--r-- | internal/fetcher/http.go (renamed from internal/importer/http.go) | 2 | ||||
| -rw-r--r-- | internal/fetcher/main.go | 74 | ||||
| -rw-r--r-- | internal/fetcher/nixpkgs-channel.go | 74 | ||||
| -rw-r--r-- | internal/importer/download-options.go | 87 | ||||
| -rw-r--r-- | internal/importer/importer.go | 81 | ||||
| -rw-r--r-- | internal/importer/main.go | 62 | ||||
| -rw-r--r-- | internal/importer/nixpkgs-channel.go | 84 | ||||
| -rw-r--r-- | internal/importer/options.go (renamed from internal/importer/ingest.go) | 6 | ||||
| -rw-r--r-- | internal/importer/package.go | 6 | ||||
| -rw-r--r-- | internal/importer/utils.go | 22 |
17 files changed, 444 insertions, 344 deletions
diff --git a/internal/config/config.go b/internal/config/config.go index eb46270..88bc006 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -115,12 +115,13 @@ var defaultConfig = Config{ Name: "NixOS", Key: "nixos", Enable: true, - Type: Channel, + Importer: Options, + Fetcher: Channel, Channel: "nixpkgs", URL: "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz", ImportPath: "nixos/release.nix", Attribute: "options", - OutputPath: "share/doc/nixos/options.json", + OutputPath: "share/doc/nixos", FetchTimeout: 5 * time.Minute, ImportTimeout: 15 * time.Minute, Repo: nixpkgs, @@ -129,12 +130,13 @@ var defaultConfig = Config{ Name: "Darwin", Key: "darwin", Enable: false, - Type: Channel, + Importer: Options, + Fetcher: Channel, Channel: "darwin", URL: "https://github.com/LnL7/nix-darwin/archive/master.tar.gz", ImportPath: "release.nix", Attribute: "options", - OutputPath: "share/doc/darwin/options.json", + OutputPath: "share/doc/darwin", FetchTimeout: 5 * time.Minute, ImportTimeout: 15 * time.Minute, Repo: Repository{ @@ -147,12 +149,13 @@ var defaultConfig = Config{ Name: "Home Manager", Key: "home-manager", Enable: false, + Importer: Options, Channel: "home-manager", URL: "https://github.com/nix-community/home-manager/archive/master.tar.gz", - Type: Channel, + Fetcher: Channel, ImportPath: "default.nix", Attribute: "docs.json", - OutputPath: "share/doc/home-manager/options.json", + OutputPath: "share/doc/home-manager", FetchTimeout: 5 * time.Minute, ImportTimeout: 15 * time.Minute, Repo: Repository{ @@ -165,7 +168,9 @@ var defaultConfig = Config{ Name: "Nix Packages", Key: "nixpkgs", Enable: true, - Type: ChannelNixpkgs, + Importer: Packages, + Fetcher: ChannelNixpkgs, + Channel: "nixos-unstable", OutputPath: "packages.json.br", FetchTimeout: 5 * time.Minute, ImportTimeout: 15 * time.Minute, diff --git a/internal/config/fetcher.go b/internal/config/fetcher.go new file mode 100644 index 0000000..5018b82 --- /dev/null +++ b/internal/config/fetcher.go @@ -0,0 +1,49 @@ +package config + +import ( + "fmt" + + "github.com/stoewer/go-strcase" +) + +type Fetcher int + +const ( + UnknownFetcher = iota + Channel + ChannelNixpkgs + Download +) + +func (f Fetcher) String() string { + switch f { + case Channel: + return "channel" + case ChannelNixpkgs: + return "channel-nixpkgs" + case Download: + return "download" + } + + return fmt.Sprintf("Fetcher(%d)", f) +} + +func parseFetcher(name string) (Fetcher, error) { + switch strcase.KebabCase(name) { + case "channel": + return Channel, nil + case "channel-nixpkgs": + return ChannelNixpkgs, nil + case "download": + return Download, nil + default: + return UnknownFetcher, fmt.Errorf("unsupported fetcher %s", name) + } +} + +func (f *Fetcher) UnmarshalText(text []byte) error { + var err error + *f, err = parseFetcher(string(text)) + + return err +} diff --git a/internal/config/importer-type.go b/internal/config/importer-type.go new file mode 100644 index 0000000..b3b3e88 --- /dev/null +++ b/internal/config/importer-type.go @@ -0,0 +1,44 @@ +package config + +import ( + "fmt" + + "github.com/stoewer/go-strcase" +) + +type ImporterType int + +const ( + UnknownType = iota + Packages + Options +) + +func (i ImporterType) String() string { + switch i { + case Packages: + return "packages" + case Options: + return "options" + } + + return fmt.Sprintf("Type(%d)", i) +} + +func parseType(name string) (ImporterType, error) { + switch strcase.KebabCase(name) { + case "packages": + return Packages, nil + case "options": + return Options, nil + default: + return UnknownType, fmt.Errorf("unsupported importer %s", name) + } +} + +func (i *ImporterType) UnmarshalText(text []byte) error { + var err error + *i, err = parseType(string(text)) + + return err +} diff --git a/internal/config/repository.go b/internal/config/repository.go index 8c17a4f..957cad4 100644 --- a/internal/config/repository.go +++ b/internal/config/repository.go @@ -8,7 +8,8 @@ import ( type RepoType int const ( - GitHub = iota + 1 + UnknownRepoType = iota + GitHub ) type Repository struct { @@ -32,7 +33,7 @@ func parseRepoType(name string) (RepoType, error) { case "github": return GitHub, nil default: - return Unknown, fmt.Errorf("unsupported repo type %s", name) + return UnknownRepoType, fmt.Errorf("unsupported repo type %s", name) } } diff --git a/internal/config/source.go b/internal/config/source.go index 9797bb1..1fab61d 100644 --- a/internal/config/source.go +++ b/internal/config/source.go @@ -1,59 +1,15 @@ package config import ( - "fmt" "time" - - "github.com/stoewer/go-strcase" -) - -type Type int - -const ( - Unknown = iota - Channel - ChannelNixpkgs - DownloadOptions ) -func (f Type) String() string { - switch f { - case Channel: - return "channel" - case ChannelNixpkgs: - return "channel-nixpkgs" - case DownloadOptions: - return "download-options" - } - - return fmt.Sprintf("Fetcher(%d)", f) -} - -func parseType(name string) (Type, error) { - switch strcase.KebabCase(name) { - case "channel": - return Channel, nil - case "channel-nixpkgs": - return ChannelNixpkgs, nil - case "download-options": - return DownloadOptions, nil - default: - return Unknown, fmt.Errorf("unsupported fetcher %s", name) - } -} - -func (f *Type) UnmarshalText(text []byte) error { - var err error - *f, err = parseType(string(text)) - - return err -} - type Source struct { Name string Key string Enable bool - Type Type + Fetcher Fetcher + Importer ImporterType Channel string URL string Attribute string diff --git a/internal/importer/channel.go b/internal/fetcher/channel.go index 1bce1b0..cadbab2 100644 --- a/internal/importer/channel.go +++ b/internal/fetcher/channel.go @@ -1,4 +1,4 @@ -package importer +package fetcher import ( "context" @@ -9,29 +9,33 @@ import ( "path" "searchix/internal/config" "searchix/internal/file" - "searchix/internal/index" "strconv" "strings" "github.com/pkg/errors" ) -type ChannelImporter struct { +type ChannelFetcher struct { DataPath string Source *config.Source SourceFile string Logger *slog.Logger } -func (i *ChannelImporter) FetchIfNeeded(parent context.Context) (bool, error) { +func (i *ChannelFetcher) FetchIfNeeded( + parent context.Context, +) (f FetchedFiles, updated bool, err error) { ctx, cancel := context.WithTimeout(parent, i.Source.FetchTimeout) defer cancel() dest := i.DataPath - before, err := os.Readlink(dest) + var before string + before, err = os.Readlink(dest) if file.NeedNotExist(err) != nil { - return false, errors.WithMessagef(err, "could not call readlink on file %s", dest) + err = errors.WithMessagef(err, "could not call readlink on file %s", dest) + + return } i.Logger.Debug("stat before", "name", before) @@ -52,9 +56,12 @@ func (i *ChannelImporter) FetchIfNeeded(parent context.Context) (bool, error) { i.Logger.Debug("nix-build command", "args", args) cmd := exec.CommandContext(ctx, "nix-build", args...) - out, err := cmd.Output() + var out []byte + out, err = cmd.Output() if err != nil { - return false, errors.WithMessage(err, "failed to run nix-build (--dry-run)") + err = errors.WithMessage(err, "failed to run nix-build (--dry-run)") + + return } i.Logger.Debug("nix-build", "output", strings.TrimSpace(string(out))) @@ -68,30 +75,25 @@ func (i *ChannelImporter) FetchIfNeeded(parent context.Context) (bool, error) { "source", i.Source.OutputPath, ) - after, err := os.Readlink(dest) - if err := file.NeedNotExist(err); err != nil { - return false, errors.WithMessagef( + var after string + after, err = os.Readlink(dest) + if err = file.NeedNotExist(err); err != nil { + err = errors.WithMessagef( err, "failed to stat output file from nix-build, filename: %s", outPath, ) + + return } i.Logger.Debug("stat after", "name", after) - return before != after, nil -} + updated = before != after -func (i *ChannelImporter) Import(parent context.Context, indexer *index.WriteIndex) (bool, error) { - if i.Source.OutputPath == "" { - return false, errors.New("no output path specified") + f = FetchedFiles{ + Options: path.Join(dest, i.Source.OutputPath, "options.json"), + Packages: path.Join(dest, i.Source.OutputPath, "packages.json"), } - filename := path.Join(i.DataPath, i.SourceFile, i.Source.OutputPath) - i.Logger.Debug("preparing import run", "revision", i.Source.Repo.Revision, "filename", filename) - - return processOptions(parent, indexer, &importConfig{ - Source: i.Source, - Filename: filename, - Logger: i.Logger, - }) + return } diff --git a/internal/fetcher/download.go b/internal/fetcher/download.go new file mode 100644 index 0000000..6bce5a8 --- /dev/null +++ b/internal/fetcher/download.go @@ -0,0 +1,75 @@ +package fetcher + +import ( + "context" + "log/slog" + "net/url" + "path" + "searchix/internal/config" + "searchix/internal/file" + + "github.com/pkg/errors" +) + +type DownloadFetcher struct { + DataPath string + Source *config.Source + SourceFile string + Logger *slog.Logger +} + +var files = map[string]string{ + "revision": "revision", + "options": "options.json", +} + +func (i *DownloadFetcher) FetchIfNeeded( + parent context.Context, +) (f FetchedFiles, updated bool, err error) { + ctx, cancel := context.WithTimeout(parent, i.Source.FetchTimeout) + defer cancel() + + root := i.DataPath + + err = file.Mkdirp(root) + if err != nil { + err = errors.WithMessagef(err, "error creating directory for data: %s", root) + + return + } + + var fetchURL string + for _, filename := range files { + fetchURL, err = url.JoinPath(i.Source.URL, filename) + if err != nil { + err = errors.WithMessagef( + err, + "could not build URL with elements %s and %s", + i.Source.URL, + filename, + ) + + return + } + + outPath := path.Join(root, filename) + + i.Logger.Debug("preparing to fetch URL", "url", fetchURL, "path", outPath) + + updated, err = fetchFileIfNeeded(ctx, outPath, fetchURL) + if err != nil { + return + } + // don't bother to issue requests for the later files + if !updated { + return + } + } + + f = FetchedFiles{ + Revision: path.Join(root, "revision"), + Options: path.Join(root, "options.json"), + } + + return +} diff --git a/internal/importer/http.go b/internal/fetcher/http.go index b496177..9afbbc0 100644 --- a/internal/importer/http.go +++ b/internal/fetcher/http.go @@ -1,4 +1,4 @@ -package importer +package fetcher import ( "context" diff --git a/internal/fetcher/main.go b/internal/fetcher/main.go new file mode 100644 index 0000000..d8bc25e --- /dev/null +++ b/internal/fetcher/main.go @@ -0,0 +1,74 @@ +package fetcher + +import ( + "context" + "log/slog" + "searchix/internal/config" + + "github.com/pkg/errors" +) + +type FetchedFiles struct { + Revision string + Options string + Packages string +} + +type Fetcher interface { + FetchIfNeeded(context.Context) (FetchedFiles, bool, error) +} + +func NewNixpkgsChannelFetcher( + source *config.Source, + dataPath string, + logger *slog.Logger, +) *NixpkgsChannelFetcher { + return &NixpkgsChannelFetcher{ + DataPath: dataPath, + Source: source, + Logger: logger, + } +} + +func NewChannelFetcher( + source *config.Source, + dataPath string, + logger *slog.Logger, +) *ChannelFetcher { + return &ChannelFetcher{ + DataPath: dataPath, + Source: source, + Logger: logger, + } +} + +func NewDownloadFetcher( + source *config.Source, + dataPath string, + logger *slog.Logger, +) *DownloadFetcher { + return &DownloadFetcher{ + DataPath: dataPath, + Source: source, + Logger: logger, + } +} + +func New( + source *config.Source, + fetcherDataPath string, + logger *slog.Logger, +) (fetcher Fetcher, err error) { + switch source.Fetcher { + case config.ChannelNixpkgs: + fetcher = NewNixpkgsChannelFetcher(source, fetcherDataPath, logger) + case config.Channel: + fetcher = NewChannelFetcher(source, fetcherDataPath, logger) + case config.Download: + fetcher = NewDownloadFetcher(source, fetcherDataPath, logger) + default: + err = errors.Errorf("unsupported fetcher type %s", source.Fetcher.String()) + } + + return +} diff --git a/internal/fetcher/nixpkgs-channel.go b/internal/fetcher/nixpkgs-channel.go new file mode 100644 index 0000000..aa1a09d --- /dev/null +++ b/internal/fetcher/nixpkgs-channel.go @@ -0,0 +1,74 @@ +package fetcher + +import ( + "context" + "log/slog" + "net/url" + "path" + "searchix/internal/config" + "searchix/internal/file" + + "github.com/pkg/errors" +) + +type NixpkgsChannelFetcher struct { + DataPath string + Source *config.Source + Logger *slog.Logger +} + +func makeChannelURL(channel string, subPath string) (string, error) { + url, err := url.JoinPath("https://channels.nixos.org/", channel, subPath) + + return url, errors.WithMessagef(err, "error creating URL") +} + +var filesToFetch = map[string]string{ + "revision": "git-revision", + "options": "options.json.br", + "packages": "packages.json.br", +} + +func (i *NixpkgsChannelFetcher) FetchIfNeeded( + parent context.Context, +) (f FetchedFiles, updated bool, err error) { + ctx, cancel := context.WithTimeout(parent, i.Source.FetchTimeout) + defer cancel() + + root := i.DataPath + + err = file.Mkdirp(root) + if err != nil { + err = errors.WithMessagef(err, "error creating directory for data: %s", root) + + return + } + + var fetchURL string + for _, filename := range filesToFetch { + fetchURL, err = makeChannelURL(i.Source.Channel, filename) + if err != nil { + return + } + + outPath := path.Join(root, filename) + + i.Logger.Debug("attempting to fetch file", "url", fetchURL, "outPath", outPath) + updated, err = fetchFileIfNeeded(ctx, outPath, fetchURL) + if err != nil { + return + } + // don't bother to issue requests for the later files + if !updated { + return + } + } + + f = FetchedFiles{ + Revision: path.Join(root, "git-revision"), + Options: path.Join(root, "options.json.br"), + Packages: path.Join(root, "packages.json.br"), + } + + return +} diff --git a/internal/importer/download-options.go b/internal/importer/download-options.go deleted file mode 100644 index 6727138..0000000 --- a/internal/importer/download-options.go +++ /dev/null @@ -1,87 +0,0 @@ -package importer - -import ( - "bytes" - "context" - "log/slog" - "net/url" - "os" - "path" - "searchix/internal/config" - "searchix/internal/file" - "searchix/internal/index" - - "github.com/pkg/errors" -) - -type DownloadOptionsImporter struct { - DataPath string - Source *config.Source - SourceFile string - Logger *slog.Logger -} - -var optionsFiles = map[string]string{ - "revision": "revision", - "options": "options.json", -} - -func (i *DownloadOptionsImporter) FetchIfNeeded(parent context.Context) (bool, error) { - ctx, cancel := context.WithTimeout(parent, i.Source.FetchTimeout) - defer cancel() - - root := i.DataPath - - err := file.Mkdirp(root) - if err != nil { - return false, errors.WithMessagef(err, "error creating directory for data: %s", root) - } - - var updated bool - for _, filename := range optionsFiles { - url, err := url.JoinPath(i.Source.URL, filename) - if err != nil { - return false, errors.WithMessagef( - err, - "could not build URL with elements %s and %s", - i.Source.URL, - filename, - ) - } - - path := path.Join(root, filename) - - i.Logger.Debug("preparing to fetch URL", "url", url, "path", path) - - updated, err = fetchFileIfNeeded(ctx, path, url) - if err != nil { - return false, err - } - // don't bother to issue requests for the later files - if !updated { - return false, err - } - } - - return updated, nil -} - -func (i *DownloadOptionsImporter) Import( - parent context.Context, - indexer *index.WriteIndex, -) (bool, error) { - filename := path.Join(i.DataPath, optionsFiles["options"]) - revFilename := path.Join(i.DataPath, optionsFiles["revision"]) - bits, err := os.ReadFile(revFilename) - if err != nil { - return false, errors.WithMessagef(err, "unable to read revision file at %s", revFilename) - } - i.Source.Repo.Revision = string(bytes.TrimSpace(bits)) - i.Logger.Debug("preparing import run", "revision", i.Source.Repo.Revision, "filename", filename) - - return processOptions(parent, indexer, &importConfig{ - Source: i.Source, - Filename: filename, - Logger: i.Logger, - }) -} diff --git a/internal/importer/importer.go b/internal/importer/importer.go index 255f70e..57118c8 100644 --- a/internal/importer/importer.go +++ b/internal/importer/importer.go @@ -3,79 +3,32 @@ package importer import ( "context" "log/slog" - "searchix/internal/config" "searchix/internal/index" + "searchix/internal/nix" "sync" ) type Importer interface { - FetchIfNeeded(context.Context) (bool, error) Import(context.Context, *index.WriteIndex) (bool, error) } -func NewNixpkgsChannelImporter( - source *config.Source, - dataPath string, - logger *slog.Logger, -) *NixpkgsChannelImporter { - return &NixpkgsChannelImporter{ - DataPath: dataPath, - Source: source, - Logger: logger, - } -} - -func NewChannelImporter( - source *config.Source, - dataPath string, - logger *slog.Logger, -) *ChannelImporter { - return &ChannelImporter{ - DataPath: dataPath, - Source: source, - Logger: logger, - } -} - -func NewDownloadOptionsImporter( - source *config.Source, - dataPath string, - logger *slog.Logger, -) *DownloadOptionsImporter { - return &DownloadOptionsImporter{ - DataPath: dataPath, - Source: source, - Logger: logger, - } -} - -type importConfig struct { - Filename string - Source *config.Source - Logger *slog.Logger +type Processor interface { + Process(context.Context) (<-chan nix.Importable, <-chan error) } -func processOptions( - parent context.Context, +func process( + ctx context.Context, indexer *index.WriteIndex, - conf *importConfig, -) (bool, error) { - ctx, cancel := context.WithTimeout(parent, conf.Source.ImportTimeout) - defer cancel() - - conf.Logger.Debug("creating option processor", "filename", conf.Filename) - processor, err := NewOptionProcessor(conf.Filename, conf.Source) - if err != nil { - return true, err - } - + processor Processor, + logger *slog.Logger, +) bool { wg := sync.WaitGroup{} wg.Add(1) - options, pErrs := processor.Process(ctx) + objects, pErrs := processor.Process(ctx) wg.Add(1) - iErrs := indexer.Import(ctx, options) + iErrs := indexer.Import(ctx, objects) var hadErrors bool go func() { @@ -85,29 +38,29 @@ func processOptions( if !running { wg.Done() iErrs = nil - conf.Logger.Info("ingest completed") + logger.Debug("ingest completed") continue } hadErrors = true - conf.Logger.Warn("error ingesting option", "error", err) + logger.Warn("error ingesting object", "error", err) case err, running := <-pErrs: if !running { wg.Done() pErrs = nil - conf.Logger.Debug("processing completed") + logger.Debug("processing completed") continue } hadErrors = true - conf.Logger.Warn("error processing option", "error", err) + logger.Warn("error processing object", "error", err) } } }() - conf.Logger.Debug("options processing", "state", "waiting") + logger.Debug("object processing", "state", "waiting") wg.Wait() - conf.Logger.Debug("options processing", "state", "complete") + logger.Debug("object processing", "state", "complete") - return hadErrors, nil + return hadErrors } diff --git a/internal/importer/main.go b/internal/importer/main.go index 0b7a99d..d2b66e1 100644 --- a/internal/importer/main.go +++ b/internal/importer/main.go @@ -2,14 +2,15 @@ package importer import ( "context" - "errors" - "log" "log/slog" "os/exec" "path" "searchix/internal/config" + "searchix/internal/fetcher" "searchix/internal/index" "strings" + + "github.com/pkg/errors" ) func Start(cfg *config.Config, indexer *index.WriteIndex, replace bool) error { @@ -22,27 +23,20 @@ func Start(cfg *config.Config, indexer *index.WriteIndex, replace bool) error { ctx, cancel := context.WithTimeout(context.Background(), cfg.Importer.Timeout.Duration) defer cancel() - var imp Importer for name, source := range cfg.Importer.Sources { - logger := slog.With("name", name, "importer", source.Type.String()) - logger.Debug("starting importer") + logger := slog.With("name", name, "fetcher", source.Fetcher.String()) + logger.Debug("starting fetcher") - importerDataPath := path.Join(cfg.DataPath, "sources", source.Channel) + fetcherDataPath := path.Join(cfg.DataPath, "sources", source.Key) - switch source.Type { - case config.ChannelNixpkgs: - imp = NewNixpkgsChannelImporter(source, importerDataPath, logger) - case config.Channel: - imp = NewChannelImporter(source, importerDataPath, logger) - case config.DownloadOptions: - imp = NewDownloadOptionsImporter(source, importerDataPath, logger) - default: - log.Printf("unsupported importer type %s", source.Type.String()) + fetcher, err := fetcher.New(source, fetcherDataPath, logger) + if err != nil { + logger.Warn("error creating fetcher", "error", err) continue } - updated, err := imp.FetchIfNeeded(ctx) + files, updated, err := fetcher.FetchIfNeeded(ctx) if err != nil { var exerr *exec.ExitError @@ -60,16 +54,38 @@ func Start(cfg *config.Config, indexer *index.WriteIndex, replace bool) error { logger.Info("importer fetch succeeded", "updated", updated) if updated || replace { - hadWarnings, err := imp.Import(ctx, indexer) - + err = setRepoRevision(files.Revision, source) if err != nil { - msg := err.Error() - for _, line := range strings.Split(strings.TrimSpace(msg), "\n") { - logger.Error("importer init failed", "error", line) - } + logger.Warn("could not set source repo revision", "error", err) + } - continue + var processor Processor + switch source.Importer { + case config.Options: + logger.Debug( + "creating processor", + "filename", + files.Options, + "revision", + source.Repo.Revision, + ) + processor, err = NewOptionProcessor(files.Options, source) + case config.Packages: + logger.Debug( + "creating processor", + "filename", + files.Packages, + "revision", + source.Repo.Revision, + ) + processor, err = NewPackageProcessor(files.Packages, source) } + if err != nil { + logger.Warn("failed to create processor", "type", source.Importer, "error", err) + } + + hadWarnings := process(ctx, indexer, processor, logger) + if hadWarnings { logger.Warn("importer succeeded, but with warnings/errors") } else { diff --git a/internal/importer/nixpkgs-channel.go b/internal/importer/nixpkgs-channel.go deleted file mode 100644 index d302154..0000000 --- a/internal/importer/nixpkgs-channel.go +++ /dev/null @@ -1,84 +0,0 @@ -package importer - -import ( - "bytes" - "context" - "log/slog" - "net/url" - "os" - "path" - "searchix/internal/config" - "searchix/internal/file" - "searchix/internal/index" - - "github.com/pkg/errors" -) - -type NixpkgsChannelImporter struct { - DataPath string - Source *config.Source - Logger *slog.Logger -} - -func makeChannelURL(channel string, subPath string) (string, error) { - url, err := url.JoinPath("https://channels.nixos.org/", channel, subPath) - - return url, errors.WithMessagef(err, "error creating URL") -} - -var filesToFetch = map[string]string{ - "revision": "git-revision", - "options": "options.json.br", -} - -func (i *NixpkgsChannelImporter) FetchIfNeeded(parent context.Context) (bool, error) { - ctx, cancel := context.WithTimeout(parent, i.Source.FetchTimeout) - defer cancel() - - root := i.DataPath - - err := file.Mkdirp(root) - if err != nil { - return false, errors.WithMessagef(err, "error creating directory for data: %s", root) - } - - for _, filename := range filesToFetch { - url, err := makeChannelURL(i.Source.Channel, filename) - if err != nil { - return false, err - } - - path := path.Join(root, filename) - - updated, err := fetchFileIfNeeded(ctx, path, url) - if err != nil { - return false, err - } - // don't bother to issue requests for the later files - if !updated { - return false, err - } - } - - return true, nil -} - -func (i *NixpkgsChannelImporter) Import( - parent context.Context, - indexer *index.WriteIndex, -) (bool, error) { - filename := path.Join(i.DataPath, filesToFetch["options"]) - revFilename := path.Join(i.DataPath, filesToFetch["revision"]) - bits, err := os.ReadFile(revFilename) - if err != nil { - return false, errors.WithMessagef(err, "unable to read revision file at %s", revFilename) - } - i.Source.Repo.Revision = string(bytes.TrimSpace(bits)) - i.Logger.Debug("preparing import run", "revision", i.Source.Repo.Revision, "filename", filename) - - return processOptions(parent, indexer, &importConfig{ - Source: i.Source, - Filename: filename, - Logger: i.Logger, - }) -} diff --git a/internal/importer/ingest.go b/internal/importer/options.go index 9b92ae8..ec2c20f 100644 --- a/internal/importer/ingest.go +++ b/internal/importer/options.go @@ -91,9 +91,8 @@ func NewOptionProcessor(inpath string, source *config.Source) (*OptionIngester, return &i, nil } -func (i *OptionIngester) Process( - ctx context.Context, -) (<-chan nix.Importable, <-chan error) { +func (i *OptionIngester) Process(parent context.Context) (<-chan nix.Importable, <-chan error) { + ctx, cancel := context.WithTimeout(parent, i.source.ImportTimeout) results := make(chan nix.Importable) errs := make(chan error) @@ -101,6 +100,7 @@ func (i *OptionIngester) Process( defer i.infile.Close() defer close(results) defer close(errs) + defer cancel() slog.Debug("starting decoder stream") outer: diff --git a/internal/importer/package.go b/internal/importer/package.go index 49e313d..3e0ec83 100644 --- a/internal/importer/package.go +++ b/internal/importer/package.go @@ -109,9 +109,8 @@ func convertToLicense(in map[string]any) *nix.License { return l } -func (i *PackageIngester) Process( - ctx context.Context, -) (<-chan nix.Importable, <-chan error) { +func (i *PackageIngester) Process(parent context.Context) (<-chan nix.Importable, <-chan error) { + ctx, cancel := context.WithTimeout(parent, i.source.ImportTimeout) results := make(chan nix.Importable) errs := make(chan error) @@ -119,6 +118,7 @@ func (i *PackageIngester) Process( defer i.infile.Close() defer close(results) defer close(errs) + defer cancel() userRepo := i.source.Repo.Owner + "/" + i.source.Repo.Repo slog.Debug("starting decoder stream") diff --git a/internal/importer/utils.go b/internal/importer/utils.go index 13d4702..3eb034f 100644 --- a/internal/importer/utils.go +++ b/internal/importer/utils.go @@ -1,11 +1,15 @@ package importer import ( + "bytes" "fmt" "net/url" + "os" + "searchix/internal/config" "searchix/internal/nix" "github.com/bcicen/jstream" + "github.com/pkg/errors" ) func ValueTypeToString(valueType jstream.ValueType) string { @@ -58,3 +62,21 @@ func MakeChannelLink(channel string, ref string, subPath string) (*nix.Link, err URL: makeGitHubFileURL(channelRepoMap[channel], ref, subPath, ""), }, nil } + +func setRepoRevision(filename string, source *config.Source) error { + if filename != "" { + bits, err := os.ReadFile(filename) + if err != nil { + return errors.WithMessagef( + err, + "unable to read revision file at %s", + filename, + ) + } + + source.Repo.Revision = string(bytes.TrimSpace(bits)) + + } + + return nil +} |