feat: enable NUR package import

6 files changed, 150 insertions, 14 deletions

diff --git a/defaults.toml b/defaults.toml
index be96d3a..f75aae6 100644
--- a/defaults.toml
+++ b/defaults.toml
@@ -95,6 +95,8 @@ ImportPath = 'release.nix'
 Timeout = '5m0s'
 # (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json.
 OutputPath = 'share/doc/darwin'
+# Depth at which packages/object object is to be found
+JSONDepth = 1
 
 # Used to generate declaration/definition links
 [Importer.Sources.darwin.Repo]
@@ -135,6 +137,8 @@ ImportPath = 'default.nix'
 Timeout = '5m0s'
 # (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json.
 OutputPath = 'share/doc/home-manager'
+# Depth at which packages/object object is to be found
+JSONDepth = 1
 
 # Used to generate declaration/definition links
 [Importer.Sources.home-manager.Repo]
@@ -175,6 +179,8 @@ ImportPath = 'nixos/release.nix'
 Timeout = '5m0s'
 # (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json.
 OutputPath = 'share/doc/nixos'
+# Depth at which packages/object object is to be found
+JSONDepth = 1
 
 # Used to generate declaration/definition links
 [Importer.Sources.nixos.Repo]
@@ -215,6 +221,8 @@ ImportPath = ''
 Timeout = '15m0s'
 # (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json.
 OutputPath = 'packages.json.br'
+# Depth at which packages/object object is to be found
+JSONDepth = 2
 
 # Used to generate declaration/definition links
 [Importer.Sources.nixpkgs.Repo]
@@ -229,3 +237,45 @@ Repo = 'nixpkgs'
 Enable = true
 # Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file
 Attribute = 'programs.sqlite'
+
+[Importer.Sources.nur]
+# Human-readable name of source for generating links
+Name = 'NUR'
+# Order in which to show source in web interface.
+Order = 4
+# Machine-readable name of source. Must be URL- and path-safe.
+Key = 'nur'
+# Controls whether to show in the web interface and to run fetch/import jobs.
+Enable = false
+# How to fetch options.json. One of 'channel', 'channel-nixpkgs' or 'download'.
+Fetcher = 'download'
+# Kind of data available from source. Currently supports 'packages' and 'options'.
+Importer = 'packages'
+# (Fetcher=channel) Local name for channel, (Fetcher=channel-nixpkgs) Remote name of channel.
+Channel = ''
+# (Fetcher=channel) Remote URL for channel, (Fetcher=download) Path containing files named 'revision' and 'options.json'.
+URL = 'https://alanpearce.github.io/nix-options/nur'
+# (Fetcher=channel) Nix attribute name (i.e. nix-build -A) that builds an {options,packages}.json
+Attribute = ''
+# (Fetcher=channel) Sub-path of imported channel which contains the attribute above, e.g. release.nix
+ImportPath = ''
+# Abort import if it takes longer than this.
+Timeout = '5m0s'
+# (Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json.
+OutputPath = ''
+# Depth at which packages/object object is to be found
+JSONDepth = 1
+
+# Used to generate declaration/definition links
+[Importer.Sources.nur.Repo]
+# Currently only 'github' is supported.
+Type = 'github'
+Owner = 'nix-community'
+Repo = 'nur'
+
+# Used to enable searching for programs in multi-program packages
+[Importer.Sources.nur.Programs]
+# Enable searching for programs in multi-program packages
+Enable = false
+# Nix attribute name (i.e. nix-instantiate) that builds a programs.sqlite file
+Attribute = ''
diff --git a/internal/config/default.go b/internal/config/default.go
index 9208a33..5260fe9 100644
--- a/internal/config/default.go
+++ b/internal/config/default.go
@@ -66,6 +66,7 @@ var DefaultConfig = Config{
 				OutputPath: "share/doc/nixos",
 				Timeout:    Duration{5 * time.Minute},
 				Repo:       nixpkgs,
+				JSONDepth:  1,
 			},
 			"darwin": {
 				Name:       "Darwin",
@@ -85,6 +86,7 @@ var DefaultConfig = Config{
 					Owner: "LnL7",
 					Repo:  "nix-darwin",
 				},
+				JSONDepth: 1,
 			},
 			"home-manager": {
 				Name:       "Home Manager",
@@ -104,6 +106,7 @@ var DefaultConfig = Config{
 					Owner: "nix-community",
 					Repo:  "home-manager",
 				},
+				JSONDepth: 1,
 			},
 			"nixpkgs": {
 				Name:       "Nix Packages",
@@ -120,6 +123,23 @@ var DefaultConfig = Config{
 					Enable:    true,
 					Attribute: "programs.sqlite",
 				},
+				JSONDepth: 2,
+			},
+			"nur": {
+				Name:     "NUR",
+				Order:    4,
+				Key:      "nur",
+				Enable:   false,
+				Importer: Packages,
+				Fetcher:  Download,
+				URL:      "https://alanpearce.github.io/nix-options/nur",
+				Timeout:  Duration{5 * time.Minute},
+				Repo: Repository{
+					Type:  GitHub,
+					Owner: "nix-community",
+					Repo:  "nur",
+				},
+				JSONDepth: 1,
 			},
 		},
 	},
diff --git a/internal/config/structs.go b/internal/config/structs.go
index 8c69733..30ad975 100644
--- a/internal/config/structs.go
+++ b/internal/config/structs.go
@@ -50,6 +50,7 @@ type Source struct {
 	OutputPath string       `comment:"(Fetcher=channel) Path under ./result symlink to folder containing {options,packages}.json."`
 	Repo       Repository   `comment:"Used to generate declaration/definition links"`
 	Programs   ProgramsDB   `comment:"Used to enable searching for programs in multi-program packages"`
+	JSONDepth  int          `comment:"Depth at which packages/object object is to be found"`
 }
 
 type ProgramsDB struct {
diff --git a/internal/fetcher/download.go b/internal/fetcher/download.go
index 6c5bae8..6dc6d34 100644
--- a/internal/fetcher/download.go
+++ b/internal/fetcher/download.go
@@ -22,7 +22,7 @@ func NewDownloadFetcher(
 	logger *log.Logger,
 ) (*DownloadFetcher, errors.E) {
 	switch source.Importer {
-	case config.Options:
+	case config.Options, config.Packages:
 		return &DownloadFetcher{
 			Source: source,
 			Logger: logger,
@@ -35,6 +35,7 @@ func NewDownloadFetcher(
 var files = map[string]string{
 	"revision": "revision",
 	"options":  "options.json",
+	"packages": "packages.json",
 }
 
 func (i *DownloadFetcher) FetchIfNeeded(
@@ -43,7 +44,17 @@ func (i *DownloadFetcher) FetchIfNeeded(
 ) (*FetchedFiles, errors.E) {
 	f := &FetchedFiles{}
 	sourceUpdated := sourceMeta.Updated
-	for key, filename := range files {
+
+	filesToFetch := make([]string, 2)
+	filesToFetch[0] = files["revision"]
+	switch i.Source.Importer {
+	case config.Packages:
+		filesToFetch[1] = files["packages"]
+	case config.Options:
+		filesToFetch[1] = files["options"]
+	}
+
+	for _, filename := range filesToFetch {
 		fetchURL, baseErr := url.JoinPath(i.Source.URL, filename)
 		if baseErr != nil {
 			return nil, errors.WithMessagef(
@@ -68,13 +79,15 @@ func (i *DownloadFetcher) FetchIfNeeded(
 		}
 		sourceMeta.Updated = mtime
 
-		switch key {
-		case "revision":
+		switch filename {
+		case files["revision"]:
 			f.Revision = body
-		case "options":
+		case files["options"]:
 			f.Options = body
+		case files["packages"]:
+			f.Packages = body
 		default:
-			return f, errors.Errorf("unknown file kind %s", key)
+			return f, errors.Errorf("unknown filename %s", filename)
 		}
 	}
 
diff --git a/internal/importer/options.go b/internal/importer/options.go
index a586a3f..695b5e2 100644
--- a/internal/importer/options.go
+++ b/internal/importer/options.go
@@ -70,7 +70,7 @@ func NewOptionProcessor(
 	log *log.Logger,
 ) (*OptionIngester, errors.E) {
 	i := OptionIngester{
-		dec:     jstream.NewDecoder(infile, 1).EmitKV(),
+		dec:     jstream.NewDecoder(infile, source.JSONDepth).EmitKV(),
 		log:     log,
 		optJSON: nixOptionJSON{},
 		infile:  infile,
diff --git a/internal/importer/package.go b/internal/importer/package.go
index 34293a7..619b10c 100644
--- a/internal/importer/package.go
+++ b/internal/importer/package.go
@@ -71,7 +71,7 @@ func NewPackageProcessor(
 	programsDB *programs.DB,
 ) (*PackageIngester, errors.E) {
 	i := &PackageIngester{
-		dec:      jstream.NewDecoder(infile, 2).EmitKV(),
+		dec:      jstream.NewDecoder(infile, source.JSONDepth).EmitKV(),
 		log:      log,
 		pkg:      packageJSON{},
 		infile:   infile,
@@ -192,18 +192,27 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, <
 
 			if meta["platforms"] != nil {
 				var plats = make([]any, len(meta["platforms"].([]any)))
-				for i, plat := range meta["platforms"].([]interface{}) {
+				i := 0
+				for _, plat := range meta["platforms"].([]interface{}) {
 					switch v := reflect.ValueOf(plat); v.Kind() {
 					case reflect.String:
 						plats[i] = v.String()
 					case reflect.Map:
 						plats[i] = makeAdhocPlatform(v.Interface())
+					case reflect.Slice:
+						ps := make([]any, v.Len())
+						for j, item := range v.Slice(0, v.Len()).Interface().([]any) {
+							ps[j] = item.(string)
+						}
+						plats = append(plats, ps...)
 					default:
 						errs <- errors.Errorf(
-							"don't know how to convert platform type %s",
+							"don't know how to convert platform type %s: %v",
 							v.Kind().String(),
+							v.Interface(),
 						)
 					}
+					i++
 				}
 				meta["platforms"] = plats
 			}
@@ -221,6 +230,43 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, <
 				}
 			}
 
+			var maints []nix.Maintainer
+			if meta["maintainers"] != nil {
+				switch maint := reflect.ValueOf(meta["maintainers"]); maint.Kind() {
+				case reflect.String:
+					maints = []nix.Maintainer{nix.Maintainer{Name: maint.String(), Github: maint.String()}}
+				case reflect.Slice, reflect.Array:
+					maints = make([]nix.Maintainer, maint.Len())
+					for i, val := range maint.Slice(0, maint.Len()).Interface().([]any) {
+						switch v := reflect.ValueOf(val); v.Kind() {
+						case reflect.String:
+							maints[i] = nix.Maintainer{Name: v.String(), Github: v.String()}
+						case reflect.Map:
+							m := v.Interface().(map[string]any)
+							maints[i] = nix.Maintainer{}
+							if m["name"] != nil && m["name"].(string) != "" {
+								maints[i].Name = m["name"].(string)
+							}
+							if m["github"] != nil && m["github"].(string) != "" {
+								maints[i].Github = m["github"].(string)
+							}
+						default:
+							errs <- errors.Errorf(
+								"don't know how to handle maintainer entry of type %s: %v",
+								v.Kind().String(),
+								v,
+							)
+						}
+					}
+				default:
+					errs <- errors.Errorf(
+						"don't know how to interpret maintainers type %s'",
+						maint.Kind().String(),
+					)
+				}
+				meta["maintainers"] = maints
+			}
+
 			i.pkg = packageJSON{}
 			if err := i.ms.Decode(x); err != nil { // stores in i.pkg
 				errs <- errors.WithMessagef(err, "failed to decode package %#v", x)
@@ -250,13 +296,19 @@ func (i *PackageIngester) Process(ctx context.Context) (<-chan nix.Importable, <
 				pkgSet = ""
 			}
 
-			url, err := makeRepoURL(i.source.Repo, subpath, line)
-			if err != nil {
-				errs <- err
+			var url string
+			if meta["position"] != nil {
+				url = meta["position"].(string)
+			} else {
+				url, err = makeRepoURL(i.source.Repo, subpath, line)
+				if err != nil {
+					errs <- err
+				}
 			}
+
 			results <- &nix.Package{
 				Name:            i.pkg.Name,
-				Attribute:       kv.Key,
+				Attribute:       strings.TrimPrefix(kv.Key, "nur.repos."),
 				Source:          i.source.Key,
 				PackageSet:      pkgSet,
 				Version:         i.pkg.Version,