feat: decode brotli-compressed files

4 files changed, 59 insertions, 16 deletions

diff --git a/internal/importer/main.go b/internal/importer/main.go
index d2b66e1..7776482 100644
--- a/internal/importer/main.go
+++ b/internal/importer/main.go
@@ -2,6 +2,7 @@ package importer
 
 import (
 	"context"
+	"io"
 	"log/slog"
 	"os/exec"
 	"path"
@@ -59,6 +60,7 @@ func Start(cfg *config.Config, indexer *index.WriteIndex, replace bool) error {
 				logger.Warn("could not set source repo revision", "error", err)
 			}
 
+			var file io.ReadCloser
 			var processor Processor
 			switch source.Importer {
 			case config.Options:
@@ -69,7 +71,11 @@ func Start(cfg *config.Config, indexer *index.WriteIndex, replace bool) error {
 					"revision",
 					source.Repo.Revision,
 				)
-				processor, err = NewOptionProcessor(files.Options, source)
+				file, err = openFileDecoded(files.Options)
+				if err != nil {
+					logger.Warn("could not open file", "filename", files.Options, "error", err)
+				}
+				processor, err = NewOptionProcessor(file, source)
 			case config.Packages:
 				logger.Debug(
 					"creating processor",
@@ -78,7 +84,11 @@ func Start(cfg *config.Config, indexer *index.WriteIndex, replace bool) error {
 					"revision",
 					source.Repo.Revision,
 				)
-				processor, err = NewPackageProcessor(files.Packages, source)
+				file, err = openFileDecoded(files.Packages)
+				if err != nil {
+					logger.Warn("could not open file", "filename", files.Packages, "error", err)
+				}
+				processor, err = NewPackageProcessor(file, source)
 			}
 			if err != nil {
 				logger.Warn("failed to create processor", "type", source.Importer, "error", err)
diff --git a/internal/importer/options.go b/internal/importer/options.go
index ec2c20f..0aadc50 100644
--- a/internal/importer/options.go
+++ b/internal/importer/options.go
@@ -2,8 +2,8 @@ package importer
 
 import (
 	"context"
+	"io"
 	"log/slog"
-	"os"
 	"reflect"
 	"searchix/internal/config"
 	"searchix/internal/nix"
@@ -58,15 +58,11 @@ type OptionIngester struct {
 	dec     *jstream.Decoder
 	ms      *mapstructure.Decoder
 	optJSON nixOptionJSON
-	infile  *os.File
+	infile  io.ReadCloser
 	source  *config.Source
 }
 
-func NewOptionProcessor(inpath string, source *config.Source) (*OptionIngester, error) {
-	infile, err := os.Open(inpath)
-	if err != nil {
-		return nil, errors.WithMessagef(err, "failed to open input file %s", inpath)
-	}
+func NewOptionProcessor(infile io.ReadCloser, source *config.Source) (*OptionIngester, error) {
 	i := OptionIngester{
 		dec:     jstream.NewDecoder(infile, 1).EmitKV(),
 		optJSON: nixOptionJSON{},
diff --git a/internal/importer/package.go b/internal/importer/package.go
index 3e0ec83..a1a6c67 100644
--- a/internal/importer/package.go
+++ b/internal/importer/package.go
@@ -3,8 +3,8 @@ package importer
 import (
 	"context"
 	"encoding/json"
+	"io"
 	"log/slog"
-	"os"
 	"reflect"
 	"searchix/internal/config"
 	"searchix/internal/nix"
@@ -41,7 +41,7 @@ type PackageIngester struct {
 	dec    *jstream.Decoder
 	ms     *mapstructure.Decoder
 	pkg    *packageJSON
-	infile *os.File
+	infile io.ReadCloser
 	source *config.Source
 }
 
@@ -60,11 +60,7 @@ func makeAdhocPlatform(v any) string {
 	return string(s)
 }
 
-func NewPackageProcessor(inpath string, source *config.Source) (*PackageIngester, error) {
-	infile, err := os.Open(inpath)
-	if err != nil {
-		return nil, errors.WithMessagef(err, "failed to open input file %s", inpath)
-	}
+func NewPackageProcessor(infile io.ReadCloser, source *config.Source) (*PackageIngester, error) {
 	i := &PackageIngester{
 		dec:    jstream.NewDecoder(infile, 2).EmitKV(),
 		pkg:    &packageJSON{},
diff --git a/internal/importer/utils.go b/internal/importer/utils.go
index 3eb034f..1c0d3af 100644
--- a/internal/importer/utils.go
+++ b/internal/importer/utils.go
@@ -3,11 +3,14 @@ package importer
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"net/url"
 	"os"
+	"path"
 	"searchix/internal/config"
 	"searchix/internal/nix"
 
+	"github.com/andybalholm/brotli"
 	"github.com/bcicen/jstream"
 	"github.com/pkg/errors"
 )
@@ -80,3 +83,41 @@ func setRepoRevision(filename string, source *config.Source) error {
 
 	return nil
 }
+
+type brotliReadCloser struct {
+	src io.ReadCloser
+	*brotli.Reader
+}
+
+func newBrotliReader(src io.ReadCloser) *brotliReadCloser {
+	return &brotliReadCloser{
+		src:    src,
+		Reader: brotli.NewReader(src),
+	}
+}
+
+func (r *brotliReadCloser) Close() error {
+	return errors.Wrap(r.src.Close(), "failed to call close on underlying reader")
+}
+
+func openFileDecoded(filename string) (io.ReadCloser, error) {
+	var reader io.ReadCloser
+	var err error
+	ext := path.Ext(filename)
+	reader, err = os.Open(filename)
+	if err != nil {
+		return nil, errors.WithMessagef(err, "failed to open file %s", filename)
+	}
+	switch ext {
+	case ".json":
+	// nothing to do
+	case ".br":
+		reader = newBrotliReader(reader)
+	default:
+		reader.Close()
+
+		return nil, errors.Errorf("invalid file extension %s", ext)
+	}
+
+	return reader, nil
+}