refactor: use a single value for per-source import timeouts
Alan Pearce alan@alanpearce.eu
Thu, 23 May 2024 13:22:50 +0200
7 files changed, 64 insertions(+), 81 deletions(-)
M defaults.toml → defaults.toml
@@ -83,10 +83,8 @@ # (Fetcher=channel) Nix attribute name (i.e. nix-build -A) that builds an {options,packages}.json Attribute = 'options' # (Fetcher=channel) Sub-path of imported channel which contains the attribute above, e.g. release.nix ImportPath = 'release.nix' -# Abort fetch if it takes longer than this. -FetchTimeout = '5m0s' # Abort import if it takes longer than this. -ImportTimeout = '15m0s' +Timeout = '5m0s' # (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json. OutputPath = 'share/doc/darwin' @@ -116,10 +114,8 @@ # (Fetcher=channel) Nix attribute name (i.e. nix-build -A) that builds an {options,packages}.json Attribute = 'docs.json' # (Fetcher=channel) Sub-path of imported channel which contains the attribute above, e.g. release.nix ImportPath = 'default.nix' -# Abort fetch if it takes longer than this. -FetchTimeout = '5m0s' # Abort import if it takes longer than this. -ImportTimeout = '15m0s' +Timeout = '5m0s' # (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json. OutputPath = 'share/doc/home-manager' @@ -149,10 +145,8 @@ # (Fetcher=channel) Nix attribute name (i.e. nix-build -A) that builds an {options,packages}.json Attribute = 'options' # (Fetcher=channel) Sub-path of imported channel which contains the attribute above, e.g. release.nix ImportPath = 'nixos/release.nix' -# Abort fetch if it takes longer than this. -FetchTimeout = '5m0s' # Abort import if it takes longer than this. -ImportTimeout = '15m0s' +Timeout = '5m0s' # (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json. OutputPath = 'share/doc/nixos' @@ -182,10 +176,8 @@ # (Fetcher=channel) Nix attribute name (i.e. nix-build -A) that builds an {options,packages}.json Attribute = '' # (Fetcher=channel) Sub-path of imported channel which contains the attribute above, e.g. release.nix ImportPath = '' -# Abort fetch if it takes longer than this. -FetchTimeout = '5m0s' # Abort import if it takes longer than this. -ImportTimeout = '15m0s' +Timeout = '5m0s' # (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json. OutputPath = 'packages.json.br'
M internal/config/config.go → internal/config/config.go
@@ -118,33 +118,31 @@ Timeout: Duration{30 * time.Minute}, UpdateAt: mustLocalTime("04:00:00"), Sources: map[string]*Source{ "nixos": { - Name: "NixOS", - Key: "nixos", - Enable: true, - Importer: Options, - Fetcher: Channel, - Channel: "nixpkgs", - URL: "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz", - ImportPath: "nixos/release.nix", - Attribute: "options", - OutputPath: "share/doc/nixos", - FetchTimeout: Duration{5 * time.Minute}, - ImportTimeout: Duration{15 * time.Minute}, - Repo: nixpkgs, + Name: "NixOS", + Key: "nixos", + Enable: true, + Importer: Options, + Fetcher: Channel, + Channel: "nixpkgs", + URL: "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz", + ImportPath: "nixos/release.nix", + Attribute: "options", + OutputPath: "share/doc/nixos", + Timeout: Duration{5 * time.Minute}, + Repo: nixpkgs, }, "darwin": { - Name: "Darwin", - Key: "darwin", - Enable: false, - Importer: Options, - Fetcher: Channel, - Channel: "darwin", - URL: "https://github.com/LnL7/nix-darwin/archive/master.tar.gz", - ImportPath: "release.nix", - Attribute: "options", - OutputPath: "share/doc/darwin", - FetchTimeout: Duration{5 * time.Minute}, - ImportTimeout: Duration{15 * time.Minute}, + Name: "Darwin", + Key: "darwin", + Enable: false, + Importer: Options, + Fetcher: Channel, + Channel: "darwin", + URL: "https://github.com/LnL7/nix-darwin/archive/master.tar.gz", + ImportPath: "release.nix", + Attribute: "options", + OutputPath: "share/doc/darwin", + Timeout: Duration{5 * time.Minute}, Repo: Repository{ Type: GitHub, Owner: "LnL7", @@ -152,18 +150,17 @@ Repo: "nix-darwin", }, }, "home-manager": { - Name: "Home Manager", - Key: "home-manager", - Enable: false, - Importer: Options, - Channel: "home-manager", - URL: "https://github.com/nix-community/home-manager/archive/master.tar.gz", - Fetcher: Channel, - ImportPath: "default.nix", - Attribute: "docs.json", - OutputPath: "share/doc/home-manager", - FetchTimeout: Duration{5 * time.Minute}, - ImportTimeout: Duration{15 * time.Minute}, + Name: "Home Manager", + Key: "home-manager", + Enable: false, + Importer: Options, + Channel: "home-manager", + URL: "https://github.com/nix-community/home-manager/archive/master.tar.gz", + Fetcher: Channel, + ImportPath: "default.nix", + Attribute: "docs.json", + OutputPath: "share/doc/home-manager", + Timeout: Duration{5 * time.Minute}, Repo: Repository{ Type: GitHub, Owner: "nix-community", @@ -171,16 +168,15 @@ Repo: "home-manager", }, }, "nixpkgs": { - Name: "Nix Packages", - Key: "nixpkgs", - Enable: true, - Importer: Packages, - Fetcher: ChannelNixpkgs, - Channel: "nixos-unstable", - OutputPath: "packages.json.br", - FetchTimeout: Duration{5 * time.Minute}, - ImportTimeout: Duration{15 * time.Minute}, - Repo: nixpkgs, + Name: "Nix Packages", + Key: "nixpkgs", + Enable: true, + Importer: Packages, + Fetcher: ChannelNixpkgs, + Channel: "nixos-unstable", + OutputPath: "packages.json.br", + Timeout: Duration{5 * time.Minute}, + Repo: nixpkgs, }, }, },
M internal/config/structs.go → internal/config/structs.go
@@ -33,17 +33,16 @@ UpdateAt LocalTime `comment:"Local time of day to run fetch/import process"` } type Source struct { - Name string `comment:"Human-readable name of source for generating links"` - Key string `comment:"Machine-readable name of source. Must be URL- and path-safe."` - Enable bool `comment:"Controls whether to show in the web interface and to run fetch/import jobs."` - Fetcher Fetcher `comment:"How to fetch options.json. One of 'channel', 'channel-nixpkgs' or 'download'."` - Importer ImporterType `comment:"Kind of data available from source. Currently supports 'packages' and 'options'."` - Channel string `comment:"(Fetcher=channel) Local name for channel, (Fetcher=channel-nixpkgs) Remote name of channel."` - URL string `comment:"(Fetcher=channel) Remote URL for channel, (Fetcher=download) Path containing files named 'revision' and 'options.json'."` - Attribute string `comment:"(Fetcher=channel) Nix attribute name (i.e. nix-build -A) that builds an {options,packages}.json"` - ImportPath string `comment:"(Fetcher=channel) Sub-path of imported channel which contains the attribute above, e.g. release.nix"` - FetchTimeout Duration `comment:"Abort fetch if it takes longer than this."` - ImportTimeout Duration `comment:"Abort import if it takes longer than this."` - OutputPath string `comment:"(Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json."` - Repo Repository `comment:"Used to generate declaration/definition links"` + Name string `comment:"Human-readable name of source for generating links"` + Key string `comment:"Machine-readable name of source. Must be URL- and path-safe."` + Enable bool `comment:"Controls whether to show in the web interface and to run fetch/import jobs."` + Fetcher Fetcher `comment:"How to fetch options.json. One of 'channel', 'channel-nixpkgs' or 'download'."` + Importer ImporterType `comment:"Kind of data available from source. Currently supports 'packages' and 'options'."` + Channel string `comment:"(Fetcher=channel) Local name for channel, (Fetcher=channel-nixpkgs) Remote name of channel."` + URL string `comment:"(Fetcher=channel) Remote URL for channel, (Fetcher=download) Path containing files named 'revision' and 'options.json'."` + Attribute string `comment:"(Fetcher=channel) Nix attribute name (i.e. nix-build -A) that builds an {options,packages}.json"` + ImportPath string `comment:"(Fetcher=channel) Sub-path of imported channel which contains the attribute above, e.g. release.nix"` + Timeout Duration `comment:"Abort import if it takes longer than this."` + OutputPath string `comment:"(Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json."` + Repo Repository `comment:"Used to generate declaration/definition links"` }
M internal/fetcher/channel.go → internal/fetcher/channel.go
@@ -44,7 +44,7 @@ ) (f FetchedFiles, err error) { args := []string{ "--no-build-output", "--timeout", - strconv.Itoa(int(i.Source.FetchTimeout.Seconds() - 1)), + strconv.Itoa(int(i.Source.Timeout.Seconds() - 1)), fmt.Sprintf("<%s/%s>", i.Source.Channel, i.Source.ImportPath), "--attr", i.Source.Attribute,
M internal/importer/main.go → internal/importer/main.go
@@ -28,7 +28,7 @@ source.Key, "fetcher", source.Fetcher.String(), "timeout", - source.FetchTimeout.Duration, + source.Timeout.Duration, ) logger.Debug("starting fetcher") @@ -42,7 +42,7 @@ if forceUpdate { sourceMeta.Updated = time.Time{} } previousUpdate := sourceMeta.Updated - ctx, cancel := context.WithTimeout(parent, source.FetchTimeout.Duration) + ctx, cancel := context.WithTimeout(parent, source.Timeout.Duration) defer cancel() files, err := fetcher.FetchIfNeeded(ctx, &sourceMeta)
M internal/importer/options.go → internal/importer/options.go
@@ -87,8 +87,7 @@ return &i, nil } -func (i *OptionIngester) Process(parent context.Context) (<-chan nix.Importable, <-chan error) { - ctx, cancel := context.WithTimeout(parent, i.source.ImportTimeout.Duration) +func (i *OptionIngester) Process(ctx context.Context) (<-chan nix.Importable, <-chan error) { results := make(chan nix.Importable) errs := make(chan error) @@ -96,7 +95,6 @@ go func() { defer i.infile.Close() defer close(results) defer close(errs) - defer cancel() outer: for mv := range i.dec.Stream() {
M internal/importer/package.go → internal/importer/package.go
@@ -104,8 +104,7 @@ return l } -func (i *PackageIngester) Process(parent context.Context) (<-chan nix.Importable, <-chan error) { - ctx, cancel := context.WithTimeout(parent, i.source.ImportTimeout.Duration) +func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, <-chan error) { results := make(chan nix.Importable) errs := make(chan error) @@ -113,7 +112,6 @@ go func() { defer i.infile.Close() defer close(results) defer close(errs) - defer cancel() outer: for mv := range i.dec.Stream() {