feat: stream files directly from fetcher to importer

Use IndexMeta to store the information relevant to making conditional
updates in future runs.

1 files changed, 8 insertions, 49 deletions

diff --git a/internal/importer/utils.go b/internal/importer/utils.go
index 425b7bd..7c53173 100644
--- a/internal/importer/utils.go
+++ b/internal/importer/utils.go
@@ -1,16 +1,13 @@
 package importer
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"net/url"
-	"os"
-	"path"
 	"searchix/internal/config"
 	"searchix/internal/nix"
+	"strings"
 
-	"github.com/andybalholm/brotli"
 	"github.com/bcicen/jstream"
 	"github.com/pkg/errors"
 )
@@ -69,58 +66,20 @@ func MakeChannelLink(repo config.Repository, subPath string) (*nix.Link, error)
 	}, nil
 }
 
-func setRepoRevision(filename string, source *config.Source) error {
-	if filename != "" {
-		bits, err := os.ReadFile(filename)
+func setRepoRevision(file io.ReadCloser, source *config.Source) error {
+	if file != nil {
+		defer file.Close()
+		var str strings.Builder
+		_, err := io.Copy(&str, file)
 		if err != nil {
 			return errors.WithMessagef(
 				err,
-				"unable to read revision file at %s",
-				filename,
+				"unable to read revision file",
 			)
 		}
 
-		source.Repo.Revision = string(bytes.TrimSpace(bits))
-
+		source.Repo.Revision = strings.TrimSpace(str.String())
 	}
 
 	return nil
 }
-
-type brotliReadCloser struct {
-	src io.ReadCloser
-	*brotli.Reader
-}
-
-func newBrotliReader(src io.ReadCloser) *brotliReadCloser {
-	return &brotliReadCloser{
-		src:    src,
-		Reader: brotli.NewReader(src),
-	}
-}
-
-func (r *brotliReadCloser) Close() error {
-	return errors.Wrap(r.src.Close(), "failed to call close on underlying reader")
-}
-
-func openFileDecoded(filename string) (io.ReadCloser, error) {
-	var reader io.ReadCloser
-	var err error
-	ext := path.Ext(filename)
-	reader, err = os.Open(filename)
-	if err != nil {
-		return nil, errors.WithMessagef(err, "failed to open file %s", filename)
-	}
-	switch ext {
-	case ".json":
-	// nothing to do
-	case ".br":
-		reader = newBrotliReader(reader)
-	default:
-		reader.Close()
-
-		return nil, errors.Errorf("invalid file extension %s", ext)
-	}
-
-	return reader, nil
-}