refactor(config): simplify configuration

commit 8225dbdb692c99b39dcafe2e5ec6ddc4daf08fb5
Author: Alan Pearce <alan@alanpearce.eu>
Date:   Mon May 13 19:18:26 2024 +0200

    refactor: consolidate configuration to reduce command-line options

commit 5616d4c5a9bc6c0c14f744f812fa6609f859dc34
Author: Alan Pearce <alan@alanpearce.eu>
Date:   Mon May 13 17:41:58 2024 +0200

    refactor: move config file parsing to program entry points

5 files changed, 215 insertions, 108 deletions

diff --git a/internal/config/config.go b/internal/config/config.go
index 340b027..c3a5a90 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -26,75 +26,108 @@ func (u *URL) UnmarshalText(text []byte) (err error) {
 	return nil
 }
 
+func mustURL(in string) (u URL) {
+	var err error
+	u.URL, err = url.Parse(in)
+	if err != nil {
+		panic(errors.Errorf("URL cannot be parsed: %s", in))
+	}
+
+	return u
+}
+
+type Web struct {
+	ContentSecurityPolicy CSP
+	ListenAddress         string
+	Port                  string
+	BaseURL               URL
+	SentryDSN             string
+	Environment           string
+	ExtraBodyHTML         template.HTML
+	Headers               map[string]string
+}
+
+type Importer struct {
+	Sources map[string]*Source
+	Timeout time.Duration
+}
+
 type Config struct {
-	DataPath      string        `toml:"data-path"`
-	CSP           CSP           `toml:"content-security-policy"`
-	ExtraBodyHTML template.HTML `toml:"extra-body-html"`
-	Headers       map[string]string
-	Sources       map[string]*Source
+	DataPath string
+	LogLevel slog.Level
+	Web      *Web
+	Importer *Importer
 }
 
 var defaultConfig = Config{
 	DataPath: "./data",
-	CSP: CSP{
-		DefaultSrc: []string{"'self'"},
-	},
-	Headers: map[string]string{
-		"x-content-type-options": "nosniff",
+	Web: &Web{
+		ListenAddress: "localhost",
+		Port:          "3000",
+		BaseURL:       mustURL("http://localhost:3000"),
+		ContentSecurityPolicy: CSP{
+			DefaultSrc: []string{"'self'"},
+		},
+		Headers: map[string]string{
+			"x-content-type-options": "nosniff",
+		},
 	},
-	Sources: map[string]*Source{
-		"nixos": {
-			Name:          "NixOS",
-			Key:           "nixos",
-			Enable:        true,
-			Type:          Channel,
-			Channel:       "nixpkgs",
-			URL:           "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz",
-			ImportPath:    "nixos/release.nix",
-			Attribute:     "options",
-			OutputPath:    "share/doc/nixos/options.json",
-			FetchTimeout:  5 * time.Minute,
-			ImportTimeout: 15 * time.Minute,
-			Repo: Repository{
-				Type:  "github",
-				Owner: "NixOS",
-				Repo:  "nixpkgs",
+	Importer: &Importer{
+		Timeout: 30 * time.Minute,
+		Sources: map[string]*Source{
+			"nixos": {
+				Name:          "NixOS",
+				Key:           "nixos",
+				Enable:        true,
+				Type:          Channel,
+				Channel:       "nixpkgs",
+				URL:           "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz",
+				ImportPath:    "nixos/release.nix",
+				Attribute:     "options",
+				OutputPath:    "share/doc/nixos/options.json",
+				FetchTimeout:  5 * time.Minute,
+				ImportTimeout: 15 * time.Minute,
+				Repo: Repository{
+					Type:  "github",
+					Owner: "NixOS",
+					Repo:  "nixpkgs",
+				},
 			},
-		},
-		"darwin": {
-			Name:          "Darwin",
-			Key:           "darwin",
-			Enable:        false,
-			Type:          Channel,
-			Channel:       "darwin",
-			URL:           "https://github.com/LnL7/nix-darwin/archive/master.tar.gz",
-			ImportPath:    "release.nix",
-			Attribute:     "options",
-			OutputPath:    "share/doc/darwin/options.json",
-			FetchTimeout:  5 * time.Minute,
-			ImportTimeout: 15 * time.Minute,
-			Repo: Repository{
-				Type:  "github",
-				Owner: "LnL7",
-				Repo:  "nix-darwin",
+			"darwin": {
+				Name:          "Darwin",
+				Key:           "darwin",
+				Enable:        false,
+				Type:          Channel,
+				Channel:       "darwin",
+				URL:           "https://github.com/LnL7/nix-darwin/archive/master.tar.gz",
+				ImportPath:    "release.nix",
+				Attribute:     "options",
+				OutputPath:    "share/doc/darwin/options.json",
+				FetchTimeout:  5 * time.Minute,
+				ImportTimeout: 15 * time.Minute,
+				Repo: Repository{
+					Type:  "github",
+					Owner: "LnL7",
+					Repo:  "nix-darwin",
+				},
 			},
-		},
-		"home-manager": {
-			Name:          "Home Manager",
-			Key:           "home-manager",
-			Enable:        false,
-			Channel:       "home-manager",
-			URL:           "https://github.com/nix-community/home-manager/archive/master.tar.gz",
-			Type:          Channel,
-			ImportPath:    "default.nix",
-			Attribute:     "docs.json",
-			OutputPath:    "share/doc/home-manager/options.json",
-			FetchTimeout:  5 * time.Minute,
-			ImportTimeout: 15 * time.Minute,
-			Repo: Repository{
-				Type:  "github",
-				Owner: "nix-community",
-				Repo:  "home-manager",
+			"home-manager": {
+				Name:          "Home Manager",
+				Key:           "home-manager",
+				Enable:        false,
+				Channel:       "home-manager",
+				URL:           "https://github.com/nix-community/home-manager/archive/master.tar.gz",
+				Type:          Channel,
+				ImportPath:    "default.nix",
+				Attribute:     "docs.json",
+				OutputPath:    "share/doc/home-manager/options.json",
+				FetchTimeout:  5 * time.Minute,
+				ImportTimeout: 15 * time.Minute,
+				Repo: Repository{
+					Type:  "github",
+					Owner: "nix-community",
+					Repo:  "home-manager",
+				},
 			},
 		},
 	},
@@ -122,7 +155,7 @@ func GetConfig(filename string) (*Config, error) {
 		}
 	}
 
-	maps.DeleteFunc(config.Sources, func(_ string, v *Source) bool {
+	maps.DeleteFunc(config.Importer.Sources, func(_ string, v *Source) bool {
 		return !v.Enable
 	})
 
diff --git a/internal/importer/main.go b/internal/importer/main.go
new file mode 100644
index 0000000..a6f15e9
--- /dev/null
+++ b/internal/importer/main.go
@@ -0,0 +1,90 @@
+package importer
+
+import (
+	"context"
+	"errors"
+	"log"
+	"log/slog"
+	"os/exec"
+	"path"
+	"searchix/internal/config"
+	"searchix/internal/search"
+	"strings"
+)
+
+func Start(cfg *config.Config, replace bool) error {
+	if len(cfg.Importer.Sources) == 0 {
+		slog.Info("No sources enabled")
+
+		return nil
+	}
+
+	indexer, err := search.NewIndexer(cfg.DataPath, replace)
+	if err != nil {
+		log.Fatalf("Failed to create indexer: %v", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), cfg.Importer.Timeout)
+	defer cancel()
+
+	var imp Importer
+	for name, source := range cfg.Importer.Sources {
+		logger := slog.With("name", name, "importer", source.Type.String())
+		logger.Debug("starting importer")
+
+		importerDataPath := path.Join(cfg.DataPath, "sources", source.Channel)
+
+		switch source.Type {
+		case config.ChannelNixpkgs:
+			imp = NewNixpkgsChannelImporter(source, importerDataPath, logger)
+		case config.Channel:
+			imp = NewChannelImporter(source, importerDataPath, logger)
+		default:
+			log.Printf("unsupported importer type %s", source.Type.String())
+
+			continue
+		}
+
+		updated, err := imp.FetchIfNeeded(ctx)
+
+		if err != nil {
+			var exerr *exec.ExitError
+			if errors.As(err, &exerr) {
+				lines := strings.Split(strings.TrimSpace(string(exerr.Stderr)), "\n")
+				for _, line := range lines {
+					logger.Warn("importer fetch failed", "stderr", line, "status", exerr.ExitCode())
+				}
+			} else {
+				logger.Warn("importer fetch failed", "error", err)
+			}
+
+			continue
+		}
+		logger.Info("importer fetch succeeded", "updated", updated)
+
+		if updated || replace {
+			hadWarnings, err := imp.Import(ctx, indexer)
+
+			if err != nil {
+				msg := err.Error()
+				for _, line := range strings.Split(strings.TrimSpace(msg), "\n") {
+					logger.Error("importer init failed", "error", line)
+				}
+
+				continue
+			}
+			if hadWarnings {
+				logger.Warn("importer succeeded, but with warnings/errors")
+			} else {
+				logger.Info("importer succeeded")
+			}
+		}
+	}
+
+	err = indexer.Close()
+	if err != nil {
+		slog.Error("error closing indexer", "error", err)
+	}
+
+	return nil
+}
diff --git a/internal/server/headers.go b/internal/server/headers.go
index 0efc384..4fb9efd 100644
--- a/internal/server/headers.go
+++ b/internal/server/headers.go
@@ -7,10 +7,10 @@ import (
 
 func AddHeadersMiddleware(next http.Handler, config *cfg.Config) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		for h, v := range config.Headers {
+		for h, v := range config.Web.Headers {
 			w.Header().Add(h, v)
 		}
-		w.Header().Add("Content-Security-Policy", config.CSP.String())
+		w.Header().Add("Content-Security-Policy", config.Web.ContentSecurityPolicy.String())
 
 		next.ServeHTTP(w, r)
 	})
diff --git a/internal/server/mux.go b/internal/server/mux.go
index ac128ce..9d3b29a 100644
--- a/internal/server/mux.go
+++ b/internal/server/mux.go
@@ -15,8 +15,7 @@ import (
 	"time"
 
 	"searchix/frontend"
-	cfg "searchix/internal/config"
-	"searchix/internal/importer"
+	"searchix/internal/config"
 	"searchix/internal/options"
 	"searchix/internal/search"
 
@@ -28,19 +27,6 @@ import (
 	"github.com/shengyanli1982/law"
 )
 
-var config *cfg.Config
-
-type Config struct {
-	Environment   string     `conf:"default:development"`
-	LiveReload    bool       `conf:"default:false,flag:live"`
-	ListenAddress string     `conf:"default:localhost"`
-	Port          string     `conf:"default:3000,short:p"`
-	BaseURL       cfg.URL    `conf:"default:http://localhost:3000,short:b"`
-	ConfigFile    string     `conf:"short:c"`
-	LogLevel      slog.Level `conf:"default:INFO"`
-	SentryDSN     string
-}
-
 type HTTPError struct {
 	Error   error
 	Message string
@@ -50,8 +36,8 @@ type HTTPError struct {
 const jsSnippet = template.HTML(livereload.JsSnippet) // #nosec G203
 
 type TemplateData struct {
-	Sources       map[string]*importer.Source
-	Source        importer.Source
+	Sources       map[string]*config.Source
+	Source        config.Source
 	Query         string
 	Results       bool
 	SourceResult  *bleve.SearchResult
@@ -67,20 +53,17 @@ type ResultData[T options.NixOption] struct {
 	Next           string
 }
 
-func applyDevModeOverrides(config *cfg.Config) {
-	if len(config.CSP.ScriptSrc) == 0 {
-		config.CSP.ScriptSrc = config.CSP.DefaultSrc
+func applyDevModeOverrides(config *config.Config) {
+	if len(config.Web.ContentSecurityPolicy.ScriptSrc) == 0 {
+		config.Web.ContentSecurityPolicy.ScriptSrc = config.Web.ContentSecurityPolicy.DefaultSrc
 	}
-	config.CSP.ScriptSrc = append(config.CSP.ScriptSrc, "'unsafe-inline'")
+	config.Web.ContentSecurityPolicy.ScriptSrc = append(
+		config.Web.ContentSecurityPolicy.ScriptSrc,
+		"'unsafe-inline'",
+	)
 }
 
-func NewMux(runtimeConfig *Config) (*http.ServeMux, error) {
-	var err error
-	config, err = cfg.GetConfig(runtimeConfig.ConfigFile)
-	if err != nil {
-		return nil, errors.WithMessage(err, "error parsing configuration file")
-	}
-
+func NewMux(config *config.Config, liveReload bool) (*http.ServeMux, error) {
 	slog.Debug("loading index")
 	index, err := search.Open(config.DataPath)
 	slog.Debug("loaded index")
@@ -91,8 +74,8 @@ func NewMux(runtimeConfig *Config) (*http.ServeMux, error) {
 	err = sentry.Init(sentry.ClientOptions{
 		EnableTracing:    true,
 		TracesSampleRate: 1.0,
-		Dsn:              runtimeConfig.SentryDSN,
-		Environment:      runtimeConfig.Environment,
+		Dsn:              config.Web.SentryDSN,
+		Environment:      config.Web.Environment,
 	})
 	if err != nil {
 		return nil, errors.WithMessage(err, "could not set up sentry")
@@ -111,8 +94,8 @@ func NewMux(runtimeConfig *Config) (*http.ServeMux, error) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/{$}", func(w http.ResponseWriter, _ *http.Request) {
 		indexData := TemplateData{
-			ExtraBodyHTML: config.ExtraBodyHTML,
-			Sources:       config.Sources,
+			ExtraBodyHTML: config.Web.ExtraBodyHTML,
+			Sources:       config.Importer.Sources,
 		}
 		err := templates["index"].ExecuteTemplate(w, "index.gotmpl", indexData)
 		if err != nil {
@@ -124,7 +107,7 @@ func NewMux(runtimeConfig *Config) (*http.ServeMux, error) {
 	mux.HandleFunc("/options/{source}/search", func(w http.ResponseWriter, r *http.Request) {
 		sourceKey := r.PathValue("source")
 
-		source := config.Sources[sourceKey]
+		source := config.Importer.Sources[sourceKey]
 		if source == nil {
 			http.Error(w, "Source not found", http.StatusNotFound)
 
@@ -157,9 +140,9 @@ func NewMux(runtimeConfig *Config) (*http.ServeMux, error) {
 
 			tdata := ResultData[options.NixOption]{
 				TemplateData: TemplateData{
-					ExtraBodyHTML: config.ExtraBodyHTML,
+					ExtraBodyHTML: config.Web.ExtraBodyHTML,
 					Source:        *source,
-					Sources:       config.Sources,
+					Sources:       config.Importer.Sources,
 				},
 				ResultsPerPage: search.ResultsPerPage,
 				Query:          qs,
@@ -216,8 +199,8 @@ func NewMux(runtimeConfig *Config) (*http.ServeMux, error) {
 			}
 
 			err = templates["search"].Execute(w, TemplateData{
-				ExtraBodyHTML: config.ExtraBodyHTML,
-				Sources:       config.Sources,
+				ExtraBodyHTML: config.Web.ExtraBodyHTML,
+				Sources:       config.Importer.Sources,
 				Source:        *source,
 				SourceResult:  sourceResult,
 			})
@@ -231,9 +214,9 @@ func NewMux(runtimeConfig *Config) (*http.ServeMux, error) {
 
 	mux.Handle("/static/", http.FileServer(http.FS(frontend.Files)))
 
-	if runtimeConfig.LiveReload {
+	if liveReload {
 		applyDevModeOverrides(config)
-		config.ExtraBodyHTML = jsSnippet
+		config.Web.ExtraBodyHTML = jsSnippet
 		liveReload := livereload.New()
 		liveReload.Start()
 		top.Handle("/livereload", liveReload)
@@ -258,7 +241,7 @@ func NewMux(runtimeConfig *Config) (*http.ServeMux, error) {
 	}
 
 	var logWriter io.Writer
-	if runtimeConfig.Environment == "production" {
+	if config.Web.Environment == "production" {
 		logWriter = law.NewWriteAsyncer(os.Stdout, nil)
 	} else {
 		logWriter = os.Stdout
@@ -267,7 +250,7 @@ func NewMux(runtimeConfig *Config) (*http.ServeMux, error) {
 		AddHeadersMiddleware(
 			sentryHandler.Handle(
 				wrapHandlerWithLogging(mux, wrappedHandlerOptions{
-					defaultHostname: runtimeConfig.BaseURL.Hostname(),
+					defaultHostname: config.Web.BaseURL.Hostname(),
 					logger:          logWriter,
 				}),
 			),
diff --git a/internal/server/server.go b/internal/server/server.go
index d13d031..77163d3 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -6,6 +6,7 @@ import (
 	"log/slog"
 	"net"
 	"net/http"
+	"searchix/internal/config"
 	"time"
 
 	"github.com/pkg/errors"
@@ -15,12 +16,12 @@ type Server struct {
 	*http.Server
 }
 
-func New(runtimeConfig *Config) (*Server, error) {
-	mux, err := NewMux(runtimeConfig)
+func New(conf *config.Config, liveReload bool) (*Server, error) {
+	mux, err := NewMux(conf, liveReload)
 	if err != nil {
 		return nil, err
 	}
-	listenAddress := net.JoinHostPort(runtimeConfig.ListenAddress, runtimeConfig.Port)
+	listenAddress := net.JoinHostPort(conf.Web.ListenAddress, conf.Web.Port)
 
 	return &Server{
 		&http.Server{