From a5e758d41c151c17ed03b39454470ba8dd0c3b99 Mon Sep 17 00:00:00 2001 From: Alan Pearce Date: Thu, 16 May 2024 23:41:57 +0200 Subject: refactor: separate fetch and import logic --- internal/fetcher/channel.go | 99 +++++++++++++++++++++++++++++++++++++ internal/fetcher/download.go | 75 ++++++++++++++++++++++++++++ internal/fetcher/http.go | 70 ++++++++++++++++++++++++++ internal/fetcher/main.go | 74 +++++++++++++++++++++++++++ internal/fetcher/nixpkgs-channel.go | 74 +++++++++++++++++++++++++++ 5 files changed, 392 insertions(+) create mode 100644 internal/fetcher/channel.go create mode 100644 internal/fetcher/download.go create mode 100644 internal/fetcher/http.go create mode 100644 internal/fetcher/main.go create mode 100644 internal/fetcher/nixpkgs-channel.go (limited to 'internal/fetcher') diff --git a/internal/fetcher/channel.go b/internal/fetcher/channel.go new file mode 100644 index 0000000..cadbab2 --- /dev/null +++ b/internal/fetcher/channel.go @@ -0,0 +1,99 @@ +package fetcher + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/exec" + "path" + "searchix/internal/config" + "searchix/internal/file" + "strconv" + "strings" + + "github.com/pkg/errors" +) + +type ChannelFetcher struct { + DataPath string + Source *config.Source + SourceFile string + Logger *slog.Logger +} + +func (i *ChannelFetcher) FetchIfNeeded( + parent context.Context, +) (f FetchedFiles, updated bool, err error) { + ctx, cancel := context.WithTimeout(parent, i.Source.FetchTimeout) + defer cancel() + + dest := i.DataPath + + var before string + before, err = os.Readlink(dest) + if file.NeedNotExist(err) != nil { + err = errors.WithMessagef(err, "could not call readlink on file %s", dest) + + return + } + i.Logger.Debug("stat before", "name", before) + + args := []string{ + "--no-build-output", + "--timeout", + strconv.Itoa(int(i.Source.FetchTimeout.Seconds() - 1)), + fmt.Sprintf("<%s/%s>", i.Source.Channel, i.Source.ImportPath), + "--attr", + i.Source.Attribute, + "--out-link", + dest, + } + + if i.Source.URL != "" { + args = append(args, "-I", fmt.Sprintf("%s=%s", i.Source.Channel, i.Source.URL)) + } + + i.Logger.Debug("nix-build command", "args", args) + cmd := exec.CommandContext(ctx, "nix-build", args...) + var out []byte + out, err = cmd.Output() + if err != nil { + err = errors.WithMessage(err, "failed to run nix-build (--dry-run)") + + return + } + i.Logger.Debug("nix-build", "output", strings.TrimSpace(string(out))) + + outPath := path.Join(dest, i.Source.OutputPath) + i.Logger.Debug( + "checking output path", + "outputPath", + outPath, + "dest", + dest, + "source", + i.Source.OutputPath, + ) + var after string + after, err = os.Readlink(dest) + if err = file.NeedNotExist(err); err != nil { + err = errors.WithMessagef( + err, + "failed to stat output file from nix-build, filename: %s", + outPath, + ) + + return + } + i.Logger.Debug("stat after", "name", after) + + updated = before != after + + f = FetchedFiles{ + Options: path.Join(dest, i.Source.OutputPath, "options.json"), + Packages: path.Join(dest, i.Source.OutputPath, "packages.json"), + } + + return +} diff --git a/internal/fetcher/download.go b/internal/fetcher/download.go new file mode 100644 index 0000000..6bce5a8 --- /dev/null +++ b/internal/fetcher/download.go @@ -0,0 +1,75 @@ +package fetcher + +import ( + "context" + "log/slog" + "net/url" + "path" + "searchix/internal/config" + "searchix/internal/file" + + "github.com/pkg/errors" +) + +type DownloadFetcher struct { + DataPath string + Source *config.Source + SourceFile string + Logger *slog.Logger +} + +var files = map[string]string{ + "revision": "revision", + "options": "options.json", +} + +func (i *DownloadFetcher) FetchIfNeeded( + parent context.Context, +) (f FetchedFiles, updated bool, err error) { + ctx, cancel := context.WithTimeout(parent, i.Source.FetchTimeout) + defer cancel() + + root := i.DataPath + + err = file.Mkdirp(root) + if err != nil { + err = errors.WithMessagef(err, "error creating directory for data: %s", root) + + return + } + + var fetchURL string + for _, filename := range files { + fetchURL, err = url.JoinPath(i.Source.URL, filename) + if err != nil { + err = errors.WithMessagef( + err, + "could not build URL with elements %s and %s", + i.Source.URL, + filename, + ) + + return + } + + outPath := path.Join(root, filename) + + i.Logger.Debug("preparing to fetch URL", "url", fetchURL, "path", outPath) + + updated, err = fetchFileIfNeeded(ctx, outPath, fetchURL) + if err != nil { + return + } + // don't bother to issue requests for the later files + if !updated { + return + } + } + + f = FetchedFiles{ + Revision: path.Join(root, "revision"), + Options: path.Join(root, "options.json"), + } + + return +} diff --git a/internal/fetcher/http.go b/internal/fetcher/http.go new file mode 100644 index 0000000..9afbbc0 --- /dev/null +++ b/internal/fetcher/http.go @@ -0,0 +1,70 @@ +package fetcher + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "os" + "searchix/internal/config" + "searchix/internal/file" + "strings" + "time" + + "github.com/pkg/errors" +) + +func fetchFileIfNeeded(ctx context.Context, path string, url string) (needed bool, err error) { + stat, err := file.StatIfExists(path) + if err != nil { + return false, errors.WithMessagef(err, "could not stat file %s", path) + } + + var mtime string + if stat != nil { + mtime = strings.Replace(stat.ModTime().UTC().Format(time.RFC1123), "UTC", "GMT", 1) + } + + req, err := http.NewRequestWithContext(ctx, "GET", url, http.NoBody) + if err != nil { + return false, errors.WithMessagef(err, "could not create HTTP request for %s", url) + } + + req.Header.Set("User-Agent", fmt.Sprintf("Searchix %s", config.ShortSHA)) + + if mtime != "" { + req.Header.Set("If-Modified-Since", mtime) + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return false, errors.WithMessagef(err, "could not make HTTP request to %s", url) + } + defer res.Body.Close() + + switch res.StatusCode { + case http.StatusNotModified: + needed = false + case http.StatusOK: + newMtime, err := time.Parse(time.RFC1123, res.Header.Get("Last-Modified")) + if err != nil { + slog.Warn( + "could not parse Last-Modified header from response", + "value", + res.Header.Get("Last-Modified"), + ) + } + err = file.WriteToFile(path, res.Body) + if err != nil { + return false, errors.WithMessagef(err, "could not write response body to file %s", path) + } + err = os.Chtimes(path, time.Time{}, newMtime) + if err != nil { + slog.Warn("could not update mtime on file", "file", path) + } + needed = true + default: + return false, fmt.Errorf("got response code %d, don't know what to do", res.StatusCode) + } + + return needed, nil +} diff --git a/internal/fetcher/main.go b/internal/fetcher/main.go new file mode 100644 index 0000000..d8bc25e --- /dev/null +++ b/internal/fetcher/main.go @@ -0,0 +1,74 @@ +package fetcher + +import ( + "context" + "log/slog" + "searchix/internal/config" + + "github.com/pkg/errors" +) + +type FetchedFiles struct { + Revision string + Options string + Packages string +} + +type Fetcher interface { + FetchIfNeeded(context.Context) (FetchedFiles, bool, error) +} + +func NewNixpkgsChannelFetcher( + source *config.Source, + dataPath string, + logger *slog.Logger, +) *NixpkgsChannelFetcher { + return &NixpkgsChannelFetcher{ + DataPath: dataPath, + Source: source, + Logger: logger, + } +} + +func NewChannelFetcher( + source *config.Source, + dataPath string, + logger *slog.Logger, +) *ChannelFetcher { + return &ChannelFetcher{ + DataPath: dataPath, + Source: source, + Logger: logger, + } +} + +func NewDownloadFetcher( + source *config.Source, + dataPath string, + logger *slog.Logger, +) *DownloadFetcher { + return &DownloadFetcher{ + DataPath: dataPath, + Source: source, + Logger: logger, + } +} + +func New( + source *config.Source, + fetcherDataPath string, + logger *slog.Logger, +) (fetcher Fetcher, err error) { + switch source.Fetcher { + case config.ChannelNixpkgs: + fetcher = NewNixpkgsChannelFetcher(source, fetcherDataPath, logger) + case config.Channel: + fetcher = NewChannelFetcher(source, fetcherDataPath, logger) + case config.Download: + fetcher = NewDownloadFetcher(source, fetcherDataPath, logger) + default: + err = errors.Errorf("unsupported fetcher type %s", source.Fetcher.String()) + } + + return +} diff --git a/internal/fetcher/nixpkgs-channel.go b/internal/fetcher/nixpkgs-channel.go new file mode 100644 index 0000000..aa1a09d --- /dev/null +++ b/internal/fetcher/nixpkgs-channel.go @@ -0,0 +1,74 @@ +package fetcher + +import ( + "context" + "log/slog" + "net/url" + "path" + "searchix/internal/config" + "searchix/internal/file" + + "github.com/pkg/errors" +) + +type NixpkgsChannelFetcher struct { + DataPath string + Source *config.Source + Logger *slog.Logger +} + +func makeChannelURL(channel string, subPath string) (string, error) { + url, err := url.JoinPath("https://channels.nixos.org/", channel, subPath) + + return url, errors.WithMessagef(err, "error creating URL") +} + +var filesToFetch = map[string]string{ + "revision": "git-revision", + "options": "options.json.br", + "packages": "packages.json.br", +} + +func (i *NixpkgsChannelFetcher) FetchIfNeeded( + parent context.Context, +) (f FetchedFiles, updated bool, err error) { + ctx, cancel := context.WithTimeout(parent, i.Source.FetchTimeout) + defer cancel() + + root := i.DataPath + + err = file.Mkdirp(root) + if err != nil { + err = errors.WithMessagef(err, "error creating directory for data: %s", root) + + return + } + + var fetchURL string + for _, filename := range filesToFetch { + fetchURL, err = makeChannelURL(i.Source.Channel, filename) + if err != nil { + return + } + + outPath := path.Join(root, filename) + + i.Logger.Debug("attempting to fetch file", "url", fetchURL, "outPath", outPath) + updated, err = fetchFileIfNeeded(ctx, outPath, fetchURL) + if err != nil { + return + } + // don't bother to issue requests for the later files + if !updated { + return + } + } + + f = FetchedFiles{ + Revision: path.Join(root, "git-revision"), + Options: path.Join(root, "options.json.br"), + Packages: path.Join(root, "packages.json.br"), + } + + return +} -- cgit 1.4.1