about summary refs log tree commit diff stats
path: root/internal/fetcher
diff options
context:
space:
mode:
Diffstat (limited to 'internal/fetcher')
-rw-r--r--internal/fetcher/channel.go99
-rw-r--r--internal/fetcher/download.go75
-rw-r--r--internal/fetcher/http.go70
-rw-r--r--internal/fetcher/main.go74
-rw-r--r--internal/fetcher/nixpkgs-channel.go74
5 files changed, 392 insertions, 0 deletions
diff --git a/internal/fetcher/channel.go b/internal/fetcher/channel.go
new file mode 100644
index 0000000..cadbab2
--- /dev/null
+++ b/internal/fetcher/channel.go
@@ -0,0 +1,99 @@
+package fetcher
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/exec"
+	"path"
+	"searchix/internal/config"
+	"searchix/internal/file"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+type ChannelFetcher struct {
+	DataPath   string
+	Source     *config.Source
+	SourceFile string
+	Logger     *slog.Logger
+}
+
+func (i *ChannelFetcher) FetchIfNeeded(
+	parent context.Context,
+) (f FetchedFiles, updated bool, err error) {
+	ctx, cancel := context.WithTimeout(parent, i.Source.FetchTimeout)
+	defer cancel()
+
+	dest := i.DataPath
+
+	var before string
+	before, err = os.Readlink(dest)
+	if file.NeedNotExist(err) != nil {
+		err = errors.WithMessagef(err, "could not call readlink on file %s", dest)
+
+		return
+	}
+	i.Logger.Debug("stat before", "name", before)
+
+	args := []string{
+		"--no-build-output",
+		"--timeout",
+		strconv.Itoa(int(i.Source.FetchTimeout.Seconds() - 1)),
+		fmt.Sprintf("<%s/%s>", i.Source.Channel, i.Source.ImportPath),
+		"--attr",
+		i.Source.Attribute,
+		"--out-link",
+		dest,
+	}
+
+	if i.Source.URL != "" {
+		args = append(args, "-I", fmt.Sprintf("%s=%s", i.Source.Channel, i.Source.URL))
+	}
+
+	i.Logger.Debug("nix-build command", "args", args)
+	cmd := exec.CommandContext(ctx, "nix-build", args...)
+	var out []byte
+	out, err = cmd.Output()
+	if err != nil {
+		err = errors.WithMessage(err, "failed to run nix-build (--dry-run)")
+
+		return
+	}
+	i.Logger.Debug("nix-build", "output", strings.TrimSpace(string(out)))
+
+	outPath := path.Join(dest, i.Source.OutputPath)
+	i.Logger.Debug(
+		"checking output path",
+		"outputPath",
+		outPath,
+		"dest",
+		dest,
+		"source",
+		i.Source.OutputPath,
+	)
+	var after string
+	after, err = os.Readlink(dest)
+	if err = file.NeedNotExist(err); err != nil {
+		err = errors.WithMessagef(
+			err,
+			"failed to stat output file from nix-build, filename: %s",
+			outPath,
+		)
+
+		return
+	}
+	i.Logger.Debug("stat after", "name", after)
+
+	updated = before != after
+
+	f = FetchedFiles{
+		Options:  path.Join(dest, i.Source.OutputPath, "options.json"),
+		Packages: path.Join(dest, i.Source.OutputPath, "packages.json"),
+	}
+
+	return
+}
diff --git a/internal/fetcher/download.go b/internal/fetcher/download.go
new file mode 100644
index 0000000..6bce5a8
--- /dev/null
+++ b/internal/fetcher/download.go
@@ -0,0 +1,75 @@
+package fetcher
+
+import (
+	"context"
+	"log/slog"
+	"net/url"
+	"path"
+	"searchix/internal/config"
+	"searchix/internal/file"
+
+	"github.com/pkg/errors"
+)
+
+type DownloadFetcher struct {
+	DataPath   string
+	Source     *config.Source
+	SourceFile string
+	Logger     *slog.Logger
+}
+
+var files = map[string]string{
+	"revision": "revision",
+	"options":  "options.json",
+}
+
+func (i *DownloadFetcher) FetchIfNeeded(
+	parent context.Context,
+) (f FetchedFiles, updated bool, err error) {
+	ctx, cancel := context.WithTimeout(parent, i.Source.FetchTimeout)
+	defer cancel()
+
+	root := i.DataPath
+
+	err = file.Mkdirp(root)
+	if err != nil {
+		err = errors.WithMessagef(err, "error creating directory for data: %s", root)
+
+		return
+	}
+
+	var fetchURL string
+	for _, filename := range files {
+		fetchURL, err = url.JoinPath(i.Source.URL, filename)
+		if err != nil {
+			err = errors.WithMessagef(
+				err,
+				"could not build URL with elements %s and %s",
+				i.Source.URL,
+				filename,
+			)
+
+			return
+		}
+
+		outPath := path.Join(root, filename)
+
+		i.Logger.Debug("preparing to fetch URL", "url", fetchURL, "path", outPath)
+
+		updated, err = fetchFileIfNeeded(ctx, outPath, fetchURL)
+		if err != nil {
+			return
+		}
+		// don't bother to issue requests for the later files
+		if !updated {
+			return
+		}
+	}
+
+	f = FetchedFiles{
+		Revision: path.Join(root, "revision"),
+		Options:  path.Join(root, "options.json"),
+	}
+
+	return
+}
diff --git a/internal/fetcher/http.go b/internal/fetcher/http.go
new file mode 100644
index 0000000..9afbbc0
--- /dev/null
+++ b/internal/fetcher/http.go
@@ -0,0 +1,70 @@
+package fetcher
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	"searchix/internal/config"
+	"searchix/internal/file"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+func fetchFileIfNeeded(ctx context.Context, path string, url string) (needed bool, err error) {
+	stat, err := file.StatIfExists(path)
+	if err != nil {
+		return false, errors.WithMessagef(err, "could not stat file %s", path)
+	}
+
+	var mtime string
+	if stat != nil {
+		mtime = strings.Replace(stat.ModTime().UTC().Format(time.RFC1123), "UTC", "GMT", 1)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", url, http.NoBody)
+	if err != nil {
+		return false, errors.WithMessagef(err, "could not create HTTP request for %s", url)
+	}
+
+	req.Header.Set("User-Agent", fmt.Sprintf("Searchix %s", config.ShortSHA))
+
+	if mtime != "" {
+		req.Header.Set("If-Modified-Since", mtime)
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return false, errors.WithMessagef(err, "could not make HTTP request to %s", url)
+	}
+	defer res.Body.Close()
+
+	switch res.StatusCode {
+	case http.StatusNotModified:
+		needed = false
+	case http.StatusOK:
+		newMtime, err := time.Parse(time.RFC1123, res.Header.Get("Last-Modified"))
+		if err != nil {
+			slog.Warn(
+				"could not parse Last-Modified header from response",
+				"value",
+				res.Header.Get("Last-Modified"),
+			)
+		}
+		err = file.WriteToFile(path, res.Body)
+		if err != nil {
+			return false, errors.WithMessagef(err, "could not write response body to file %s", path)
+		}
+		err = os.Chtimes(path, time.Time{}, newMtime)
+		if err != nil {
+			slog.Warn("could not update mtime on file", "file", path)
+		}
+		needed = true
+	default:
+		return false, fmt.Errorf("got response code %d, don't know what to do", res.StatusCode)
+	}
+
+	return needed, nil
+}
diff --git a/internal/fetcher/main.go b/internal/fetcher/main.go
new file mode 100644
index 0000000..d8bc25e
--- /dev/null
+++ b/internal/fetcher/main.go
@@ -0,0 +1,74 @@
+package fetcher
+
+import (
+	"context"
+	"log/slog"
+	"searchix/internal/config"
+
+	"github.com/pkg/errors"
+)
+
+type FetchedFiles struct {
+	Revision string
+	Options  string
+	Packages string
+}
+
+type Fetcher interface {
+	FetchIfNeeded(context.Context) (FetchedFiles, bool, error)
+}
+
+func NewNixpkgsChannelFetcher(
+	source *config.Source,
+	dataPath string,
+	logger *slog.Logger,
+) *NixpkgsChannelFetcher {
+	return &NixpkgsChannelFetcher{
+		DataPath: dataPath,
+		Source:   source,
+		Logger:   logger,
+	}
+}
+
+func NewChannelFetcher(
+	source *config.Source,
+	dataPath string,
+	logger *slog.Logger,
+) *ChannelFetcher {
+	return &ChannelFetcher{
+		DataPath: dataPath,
+		Source:   source,
+		Logger:   logger,
+	}
+}
+
+func NewDownloadFetcher(
+	source *config.Source,
+	dataPath string,
+	logger *slog.Logger,
+) *DownloadFetcher {
+	return &DownloadFetcher{
+		DataPath: dataPath,
+		Source:   source,
+		Logger:   logger,
+	}
+}
+
+func New(
+	source *config.Source,
+	fetcherDataPath string,
+	logger *slog.Logger,
+) (fetcher Fetcher, err error) {
+	switch source.Fetcher {
+	case config.ChannelNixpkgs:
+		fetcher = NewNixpkgsChannelFetcher(source, fetcherDataPath, logger)
+	case config.Channel:
+		fetcher = NewChannelFetcher(source, fetcherDataPath, logger)
+	case config.Download:
+		fetcher = NewDownloadFetcher(source, fetcherDataPath, logger)
+	default:
+		err = errors.Errorf("unsupported fetcher type %s", source.Fetcher.String())
+	}
+
+	return
+}
diff --git a/internal/fetcher/nixpkgs-channel.go b/internal/fetcher/nixpkgs-channel.go
new file mode 100644
index 0000000..aa1a09d
--- /dev/null
+++ b/internal/fetcher/nixpkgs-channel.go
@@ -0,0 +1,74 @@
+package fetcher
+
+import (
+	"context"
+	"log/slog"
+	"net/url"
+	"path"
+	"searchix/internal/config"
+	"searchix/internal/file"
+
+	"github.com/pkg/errors"
+)
+
+type NixpkgsChannelFetcher struct {
+	DataPath string
+	Source   *config.Source
+	Logger   *slog.Logger
+}
+
+func makeChannelURL(channel string, subPath string) (string, error) {
+	url, err := url.JoinPath("https://channels.nixos.org/", channel, subPath)
+
+	return url, errors.WithMessagef(err, "error creating URL")
+}
+
+var filesToFetch = map[string]string{
+	"revision": "git-revision",
+	"options":  "options.json.br",
+	"packages": "packages.json.br",
+}
+
+func (i *NixpkgsChannelFetcher) FetchIfNeeded(
+	parent context.Context,
+) (f FetchedFiles, updated bool, err error) {
+	ctx, cancel := context.WithTimeout(parent, i.Source.FetchTimeout)
+	defer cancel()
+
+	root := i.DataPath
+
+	err = file.Mkdirp(root)
+	if err != nil {
+		err = errors.WithMessagef(err, "error creating directory for data: %s", root)
+
+		return
+	}
+
+	var fetchURL string
+	for _, filename := range filesToFetch {
+		fetchURL, err = makeChannelURL(i.Source.Channel, filename)
+		if err != nil {
+			return
+		}
+
+		outPath := path.Join(root, filename)
+
+		i.Logger.Debug("attempting to fetch file", "url", fetchURL, "outPath", outPath)
+		updated, err = fetchFileIfNeeded(ctx, outPath, fetchURL)
+		if err != nil {
+			return
+		}
+		// don't bother to issue requests for the later files
+		if !updated {
+			return
+		}
+	}
+
+	f = FetchedFiles{
+		Revision: path.Join(root, "git-revision"),
+		Options:  path.Join(root, "options.json.br"),
+		Packages: path.Join(root, "packages.json.br"),
+	}
+
+	return
+}
generated by cgit-pink 1.4.1 (git 2.36.1) at 2024-11-22 10:53:56 +0000