From e062ca72b222b890e345548bd8422d5df98e9fef Mon Sep 17 00:00:00 2001 From: Alan Pearce Date: Thu, 9 May 2024 16:47:41 +0200 Subject: feat: import sources from configuration in go code and index options --- internal/importer/ingest.go | 237 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 internal/importer/ingest.go (limited to 'internal/importer/ingest.go') diff --git a/internal/importer/ingest.go b/internal/importer/ingest.go new file mode 100644 index 0000000..b9db80c --- /dev/null +++ b/internal/importer/ingest.go @@ -0,0 +1,237 @@ +package importer + +import ( + "context" + "fmt" + "log/slog" + "net/url" + "os" + "reflect" + "searchix/internal/options" + "strings" + + "github.com/bcicen/jstream" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +type nixValueJSON struct { + Type string `mapstructure:"_type"` + Text string +} + +type linkJSON struct { + Name string + URL string `json:"url"` +} + +type nixOptionJSON struct { + Declarations []linkJSON + Default *nixValueJSON + Description string + Example *nixValueJSON + Loc []string + ReadOnly bool + RelatedPackages string + Type string +} + +func ValueTypeToString(valueType jstream.ValueType) string { + switch valueType { + case jstream.Unknown: + return "unknown" + case jstream.Null: + return "null" + case jstream.String: + return "string" + case jstream.Number: + return "number" + case jstream.Boolean: + return "boolean" + case jstream.Array: + return "array" + case jstream.Object: + return "object" + } + + return "very strange" +} + +func makeGitHubFileURL(userRepo string, ref string, subPath string) string { + url, _ := url.JoinPath("https://github.com/", userRepo, "blob", ref, subPath) + + return url +} + +// make configurable? +var channelRepoMap = map[string]string{ + "nixpkgs": "NixOS/nixpkgs", + "nix-darwin": "LnL7/nix-darwin", + "home-manager": "nix-community/home-manager", +} + +func MakeChannelLink(channel string, ref string, subPath string) (*options.Link, error) { + if channelRepoMap[channel] == "" { + return nil, fmt.Errorf("don't know what repository relates to channel <%s>", channel) + } + + return &options.Link{ + Name: fmt.Sprintf("<%s/%s>", channel, subPath), + URL: makeGitHubFileURL(channelRepoMap[channel], ref, subPath), + }, nil +} + +func convertNixValue(nj *nixValueJSON) *options.NixValue { + if nj == nil { + return nil + } + switch nj.Type { + case "", "literalExpression": + return &options.NixValue{ + Text: nj.Text, + } + case "literalMD": + return &options.NixValue{ + Markdown: options.Markdown(nj.Text), + } + default: + slog.Warn("got unexpected NixValue type", "type", nj.Type, "text", nj.Text) + + return nil + } +} + +type OptionIngester struct { + dec *jstream.Decoder + ms *mapstructure.Decoder + optJSON nixOptionJSON + infile *os.File + source Source +} + +type Ingester[T options.NixOption] interface { + Process() (<-chan *T, <-chan error) +} + +func NewOptionProcessor(inpath string, source Source) (*OptionIngester, error) { + infile, err := os.Open(inpath) + if err != nil { + return nil, errors.WithMessagef(err, "failed to open input file %s", inpath) + } + i := OptionIngester{ + dec: jstream.NewDecoder(infile, 1).EmitKV(), + optJSON: nixOptionJSON{}, + infile: infile, + source: source, + } + + ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + ErrorUnused: true, + ZeroFields: true, + Result: &i.optJSON, + Squash: true, + DecodeHook: mapstructure.TextUnmarshallerHookFunc(), + }) + if err != nil { + defer infile.Close() + + return nil, errors.WithMessage(err, "could not create mapstructure decoder") + } + i.ms = ms + + return &i, nil +} + +func (i *OptionIngester) Process(ctx context.Context) (<-chan *options.NixOption, <-chan error) { + results := make(chan *options.NixOption) + errs := make(chan error) + + go func() { + defer i.infile.Close() + defer close(results) + defer close(errs) + + slog.Debug("starting decoder stream") + outer: + for mv := range i.dec.Stream() { + select { + case <-ctx.Done(): + break outer + default: + } + if err := i.dec.Err(); err != nil { + errs <- errors.WithMessage(err, "could not decode JSON") + + continue + } + if mv.ValueType != jstream.Object { + errs <- errors.Errorf("unexpected object type %s", ValueTypeToString(mv.ValueType)) + + continue + } + kv := mv.Value.(jstream.KV) + x := kv.Value.(map[string]interface{}) + + var decls []*options.Link + for _, decl := range x["declarations"].([]interface{}) { + i.optJSON = nixOptionJSON{} + + switch decl := reflect.ValueOf(decl); decl.Kind() { + case reflect.String: + s := decl.String() + link, err := MakeChannelLink(i.source.Channel, i.source.Repo.Revision, s) + if err != nil { + errs <- errors.WithMessagef(err, + "could not make a channel link for channel %s, revision %s and subpath %s", + i.source.Channel, i.source.Repo.Revision, s, + ) + + continue + } + decls = append(decls, link) + case reflect.Map: + v := decl.Interface().(map[string]interface{}) + link := options.Link{ + Name: v["name"].(string), + URL: v["url"].(string), + } + decls = append(decls, &link) + default: + errs <- errors.Errorf("unexpected declaration type %s", decl.Kind().String()) + + continue + } + } + if len(decls) > 0 { + x["declarations"] = decls + } + + err := i.ms.Decode(x) // stores in optJSON + if err != nil { + errs <- errors.WithMessagef(err, "failed to decode option %#v", x) + + continue + } + + var decs = make([]options.Link, len(i.optJSON.Declarations)) + for i, d := range i.optJSON.Declarations { + decs[i] = options.Link(d) + } + + // slog.Debug("sending option", "name", kv.Key) + results <- &options.NixOption{ + Option: kv.Key, + Source: strings.ToLower(i.source.Name), + Declarations: decs, + Default: convertNixValue(i.optJSON.Default), + Description: options.Markdown(i.optJSON.Description), + Example: convertNixValue(i.optJSON.Example), + RelatedPackages: options.Markdown(i.optJSON.RelatedPackages), + Loc: i.optJSON.Loc, + Type: i.optJSON.Type, + } + } + }() + + return results, errs +} -- cgit 1.4.1