feat: enable NUR package import
6 files changed, 150 insertions(+), 14 deletions(-)
M defaults.toml → defaults.toml
@@ -95,6 +95,8 @@ # Abort import if it takes longer than this. Timeout = '5m0s' # (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json. OutputPath = 'share/doc/darwin' +# Depth at which packages/object object is to be found +JSONDepth = 1 # Used to generate declaration/definition links [Importer.Sources.darwin.Repo]@@ -135,6 +137,8 @@ # Abort import if it takes longer than this. Timeout = '5m0s' # (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json. OutputPath = 'share/doc/home-manager' +# Depth at which packages/object object is to be found +JSONDepth = 1 # Used to generate declaration/definition links [Importer.Sources.home-manager.Repo]@@ -175,6 +179,8 @@ # Abort import if it takes longer than this. Timeout = '5m0s' # (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json. OutputPath = 'share/doc/nixos' +# Depth at which packages/object object is to be found +JSONDepth = 1 # Used to generate declaration/definition links [Importer.Sources.nixos.Repo]@@ -215,6 +221,8 @@ # Abort import if it takes longer than this. Timeout = '15m0s' # (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json. OutputPath = 'packages.json.br' +# Depth at which packages/object object is to be found +JSONDepth = 2 # Used to generate declaration/definition links [Importer.Sources.nixpkgs.Repo]@@ -229,3 +237,45 @@ # Enable searching for programs in multi-program packages Enable = true # Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file Attribute = 'programs.sqlite' + +[Importer.Sources.nur] +# Human-readable name of source for generating links +Name = 'NUR' +# Order in which to show source in web interface. +Order = 4 +# Machine-readable name of source. Must be URL- and path-safe. +Key = 'nur' +# Controls whether to show in the web interface and to run fetch/import jobs. +Enable = false +# How to fetch options.json. One of 'channel', 'channel-nixpkgs' or 'download'. +Fetcher = 'download' +# Kind of data available from source. Currently supports 'packages' and 'options'. +Importer = 'packages' +# (Fetcher=channel) Local name for channel, (Fetcher=channel-nixpkgs) Remote name of channel. +Channel = '' +# (Fetcher=channel) Remote URL for channel, (Fetcher=download) Path containing files named 'revision' and 'options.json'. +URL = 'https://alanpearce.github.io/nix-options/nur' +# (Fetcher=channel) Nix attribute name (i.e. nix-build -A) that builds an {options,packages}.json +Attribute = '' +# (Fetcher=channel) Sub-path of imported channel which contains the attribute above, e.g. release.nix +ImportPath = '' +# Abort import if it takes longer than this. +Timeout = '5m0s' +# (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json. +OutputPath = '' +# Depth at which packages/object object is to be found +JSONDepth = 1 + +# Used to generate declaration/definition links +[Importer.Sources.nur.Repo] +# Currently only 'github' is supported. +Type = 'github' +Owner = 'nix-community' +Repo = 'nur' + +# Used to enable searching for programs in multi-program packages +[Importer.Sources.nur.Programs] +# Enable searching for programs in multi-program packages +Enable = false +# Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file +Attribute = ''
M internal/config/default.go → internal/config/default.go
@@ -66,6 +66,7 @@ Attribute: "options", OutputPath: "share/doc/nixos", Timeout: Duration{5 * time.Minute}, Repo: nixpkgs, + JSONDepth: 1, }, "darwin": { Name: "Darwin",@@ -85,6 +86,7 @@ Type: GitHub, Owner: "LnL7", Repo: "nix-darwin", }, + JSONDepth: 1, }, "home-manager": { Name: "Home Manager",@@ -104,6 +106,7 @@ Type: GitHub, Owner: "nix-community", Repo: "home-manager", }, + JSONDepth: 1, }, "nixpkgs": { Name: "Nix Packages",@@ -120,6 +123,23 @@ Programs: ProgramsDB{ Enable: true, Attribute: "programs.sqlite", }, + JSONDepth: 2, + }, + "nur": { + Name: "NUR", + Order: 4, + Key: "nur", + Enable: false, + Importer: Packages, + Fetcher: Download, + URL: "https://alanpearce.github.io/nix-options/nur", + Timeout: Duration{5 * time.Minute}, + Repo: Repository{ + Type: GitHub, + Owner: "nix-community", + Repo: "nur", + }, + JSONDepth: 1, }, }, },
M internal/config/structs.go → internal/config/structs.go
@@ -50,6 +50,7 @@ Timeout Duration `comment:"Abort import if it takes longer than this."` OutputPath string `comment:"(Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json."` Repo Repository `comment:"Used to generate declaration/definition links"` Programs ProgramsDB `comment:"Used to enable searching for programs in multi-program packages"` + JSONDepth int `comment:"Depth at which packages/object object is to be found"` } type ProgramsDB struct {
M internal/fetcher/download.go → internal/fetcher/download.go
@@ -22,7 +22,7 @@ source *config.Source, logger *log.Logger, ) (*DownloadFetcher, errors.E) { switch source.Importer { - case config.Options: + case config.Options, config.Packages: return &DownloadFetcher{ Source: source, Logger: logger,@@ -35,6 +35,7 @@ var files = map[string]string{ "revision": "revision", "options": "options.json", + "packages": "packages.json", } func (i *DownloadFetcher) FetchIfNeeded(@@ -43,7 +44,17 @@ sourceMeta *index.SourceMeta, ) (*FetchedFiles, errors.E) { f := &FetchedFiles{} sourceUpdated := sourceMeta.Updated - for key, filename := range files { + + filesToFetch := make([]string, 2) + filesToFetch[0] = files["revision"] + switch i.Source.Importer { + case config.Packages: + filesToFetch[1] = files["packages"] + case config.Options: + filesToFetch[1] = files["options"] + } + + for _, filename := range filesToFetch { fetchURL, baseErr := url.JoinPath(i.Source.URL, filename) if baseErr != nil { return nil, errors.WithMessagef(@@ -68,13 +79,15 @@ break } sourceMeta.Updated = mtime - switch key { - case "revision": + switch filename { + case files["revision"]: f.Revision = body - case "options": + case files["options"]: f.Options = body + case files["packages"]: + f.Packages = body default: - return f, errors.Errorf("unknown file kind %s", key) + return f, errors.Errorf("unknown filename %s", filename) } }
M internal/importer/options.go → internal/importer/options.go
@@ -70,7 +70,7 @@ source *config.Source, log *log.Logger, ) (*OptionIngester, errors.E) { i := OptionIngester{ - dec: jstream.NewDecoder(infile, 1).EmitKV(), + dec: jstream.NewDecoder(infile, source.JSONDepth).EmitKV(), log: log, optJSON: nixOptionJSON{}, infile: infile,
M internal/importer/package.go → internal/importer/package.go
@@ -71,7 +71,7 @@ log *log.Logger, programsDB *programs.DB, ) (*PackageIngester, errors.E) { i := &PackageIngester{ - dec: jstream.NewDecoder(infile, 2).EmitKV(), + dec: jstream.NewDecoder(infile, source.JSONDepth).EmitKV(), log: log, pkg: packageJSON{}, infile: infile,@@ -192,18 +192,27 @@ } if meta["platforms"] != nil { var plats = make([]any, len(meta["platforms"].([]any))) - for i, plat := range meta["platforms"].([]interface{}) { + i := 0 + for _, plat := range meta["platforms"].([]interface{}) { switch v := reflect.ValueOf(plat); v.Kind() { case reflect.String: plats[i] = v.String() case reflect.Map: plats[i] = makeAdhocPlatform(v.Interface()) + case reflect.Slice: + ps := make([]any, v.Len()) + for j, item := range v.Slice(0, v.Len()).Interface().([]any) { + ps[j] = item.(string) + } + plats = append(plats, ps...) default: errs <- errors.Errorf( - "don't know how to convert platform type %s", + "don't know how to convert platform type %s: %v", v.Kind().String(), + v.Interface(), ) } + i++ } meta["platforms"] = plats }@@ -221,6 +230,43 @@ ) } } + var maints []nix.Maintainer + if meta["maintainers"] != nil { + switch maint := reflect.ValueOf(meta["maintainers"]); maint.Kind() { + case reflect.String: + maints = []nix.Maintainer{nix.Maintainer{Name: maint.String(), Github: maint.String()}} + case reflect.Slice, reflect.Array: + maints = make([]nix.Maintainer, maint.Len()) + for i, val := range maint.Slice(0, maint.Len()).Interface().([]any) { + switch v := reflect.ValueOf(val); v.Kind() { + case reflect.String: + maints[i] = nix.Maintainer{Name: v.String(), Github: v.String()} + case reflect.Map: + m := v.Interface().(map[string]any) + maints[i] = nix.Maintainer{} + if m["name"] != nil && m["name"].(string) != "" { + maints[i].Name = m["name"].(string) + } + if m["github"] != nil && m["github"].(string) != "" { + maints[i].Github = m["github"].(string) + } + default: + errs <- errors.Errorf( + "don't know how to handle maintainer entry of type %s: %v", + v.Kind().String(), + v, + ) + } + } + default: + errs <- errors.Errorf( + "don't know how to interpret maintainers type %s'", + maint.Kind().String(), + ) + } + meta["maintainers"] = maints + } + i.pkg = packageJSON{} if err := i.ms.Decode(x); err != nil { // stores in i.pkg errs <- errors.WithMessagef(err, "failed to decode package %#v", x)@@ -250,13 +296,19 @@ if !found { pkgSet = "" } - url, err := makeRepoURL(i.source.Repo, subpath, line) - if err != nil { - errs <- err + var url string + if meta["position"] != nil { + url = meta["position"].(string) + } else { + url, err = makeRepoURL(i.source.Repo, subpath, line) + if err != nil { + errs <- err + } } + results <- &nix.Package{ Name: i.pkg.Name, - Attribute: kv.Key, + Attribute: strings.TrimPrefix(kv.Key, "nur.repos."), Source: i.source.Key, PackageSet: pkgSet, Version: i.pkg.Version,