diff options
author | Alan Pearce | 2024-05-13 21:50:14 +0200 |
---|---|---|
committer | Alan Pearce | 2024-05-13 22:25:46 +0200 |
commit | 6b40e0c0fa90f11be14a93f1d6275779fd645cac (patch) | |
tree | 0df5bea87242450829ca441d235c077a0e09a149 /internal | |
parent | 2722f24af87f437ed9fcb8cc743ad1784141fd3a (diff) | |
download | searchix-6b40e0c0fa90f11be14a93f1d6275779fd645cac.tar.lz searchix-6b40e0c0fa90f11be14a93f1d6275779fd645cac.tar.zst searchix-6b40e0c0fa90f11be14a93f1d6275779fd645cac.zip |
refactor: combine import and web server into one binary
Diffstat (limited to 'internal')
-rw-r--r-- | internal/config/config.go | 30 | ||||
-rw-r--r-- | internal/importer/channel.go | 4 | ||||
-rw-r--r-- | internal/importer/importer.go | 6 | ||||
-rw-r--r-- | internal/importer/main.go | 16 | ||||
-rw-r--r-- | internal/importer/nixpkgs-channel.go | 4 | ||||
-rw-r--r-- | internal/index/index_meta.go (renamed from internal/search/index_meta.go) | 16 | ||||
-rw-r--r-- | internal/index/indexer.go (renamed from internal/search/indexer.go) | 40 | ||||
-rw-r--r-- | internal/index/search.go (renamed from internal/search/search.go) | 25 | ||||
-rw-r--r-- | internal/server/mux.go | 17 | ||||
-rw-r--r-- | internal/server/server.go | 5 |
10 files changed, 80 insertions, 83 deletions
diff --git a/internal/config/config.go b/internal/config/config.go index c3a5a90..ec04a2c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -26,6 +26,19 @@ func (u *URL) UnmarshalText(text []byte) (err error) { return nil } +type Duration struct { + time.Duration +} + +func (d *Duration) UnmarshalText(text []byte) (err error) { + d.Duration, err = time.ParseDuration(string(text)) + if err != nil { + return errors.WithMessagef(err, "could not parse duration %s", string(text)) + } + + return nil +} + func mustURL(in string) (u URL) { var err error u.URL, err = url.Parse(in) @@ -36,6 +49,15 @@ func mustURL(in string) (u URL) { return u } +func mustLocalTime(in string) (time toml.LocalTime) { + err := time.UnmarshalText([]byte(in)) + if err != nil { + panic(errors.Errorf("Could not parse time: %s", in)) + } + + return +} + type Web struct { ContentSecurityPolicy CSP ListenAddress string @@ -48,8 +70,9 @@ type Web struct { } type Importer struct { - Sources map[string]*Source - Timeout time.Duration + Sources map[string]*Source + Timeout Duration + UpdateAt toml.LocalTime } type Config struct { @@ -73,7 +96,8 @@ var defaultConfig = Config{ }, }, Importer: &Importer{ - Timeout: 30 * time.Minute, + Timeout: Duration{30 * time.Minute}, + UpdateAt: mustLocalTime("04:00:00"), Sources: map[string]*Source{ "nixos": { Name: "NixOS", diff --git a/internal/importer/channel.go b/internal/importer/channel.go index fb6668c..1bce1b0 100644 --- a/internal/importer/channel.go +++ b/internal/importer/channel.go @@ -9,7 +9,7 @@ import ( "path" "searchix/internal/config" "searchix/internal/file" - "searchix/internal/search" + "searchix/internal/index" "strconv" "strings" @@ -81,7 +81,7 @@ func (i *ChannelImporter) FetchIfNeeded(parent context.Context) (bool, error) { return before != after, nil } -func (i *ChannelImporter) Import(parent context.Context, indexer *search.WriteIndex) (bool, error) { +func (i *ChannelImporter) Import(parent context.Context, indexer *index.WriteIndex) (bool, error) { if i.Source.OutputPath == "" { return false, errors.New("no output path specified") } diff --git a/internal/importer/importer.go b/internal/importer/importer.go index 5f251b0..a242481 100644 --- a/internal/importer/importer.go +++ b/internal/importer/importer.go @@ -5,13 +5,13 @@ import ( "log/slog" "path" "searchix/internal/config" - "searchix/internal/search" + "searchix/internal/index" "sync" ) type Importer interface { FetchIfNeeded(context.Context) (bool, error) - Import(context.Context, *search.WriteIndex) (bool, error) + Import(context.Context, *index.WriteIndex) (bool, error) } func NewNixpkgsChannelImporter( @@ -50,7 +50,7 @@ type importConfig struct { func processOptions( parent context.Context, - indexer *search.WriteIndex, + indexer *index.WriteIndex, conf *importConfig, ) (bool, error) { ctx, cancel := context.WithTimeout(parent, conf.Source.ImportTimeout) diff --git a/internal/importer/main.go b/internal/importer/main.go index a6f15e9..2d87e33 100644 --- a/internal/importer/main.go +++ b/internal/importer/main.go @@ -8,23 +8,18 @@ import ( "os/exec" "path" "searchix/internal/config" - "searchix/internal/search" + "searchix/internal/index" "strings" ) -func Start(cfg *config.Config, replace bool) error { +func Start(cfg *config.Config, indexer *index.WriteIndex, replace bool) error { if len(cfg.Importer.Sources) == 0 { slog.Info("No sources enabled") return nil } - indexer, err := search.NewIndexer(cfg.DataPath, replace) - if err != nil { - log.Fatalf("Failed to create indexer: %v", err) - } - - ctx, cancel := context.WithTimeout(context.Background(), cfg.Importer.Timeout) + ctx, cancel := context.WithTimeout(context.Background(), cfg.Importer.Timeout.Duration) defer cancel() var imp Importer @@ -81,10 +76,5 @@ func Start(cfg *config.Config, replace bool) error { } } - err = indexer.Close() - if err != nil { - slog.Error("error closing indexer", "error", err) - } - return nil } diff --git a/internal/importer/nixpkgs-channel.go b/internal/importer/nixpkgs-channel.go index 7aaa816..d302154 100644 --- a/internal/importer/nixpkgs-channel.go +++ b/internal/importer/nixpkgs-channel.go @@ -9,7 +9,7 @@ import ( "path" "searchix/internal/config" "searchix/internal/file" - "searchix/internal/search" + "searchix/internal/index" "github.com/pkg/errors" ) @@ -65,7 +65,7 @@ func (i *NixpkgsChannelImporter) FetchIfNeeded(parent context.Context) (bool, er func (i *NixpkgsChannelImporter) Import( parent context.Context, - indexer *search.WriteIndex, + indexer *index.WriteIndex, ) (bool, error) { filename := path.Join(i.DataPath, filesToFetch["options"]) revFilename := path.Join(i.DataPath, filesToFetch["revision"]) diff --git a/internal/search/index_meta.go b/internal/index/index_meta.go index bb7e69f..e24cd3b 100644 --- a/internal/search/index_meta.go +++ b/internal/index/index_meta.go @@ -1,4 +1,4 @@ -package search +package index import ( "encoding/json" @@ -11,12 +11,12 @@ import ( const CurrentSchemaVersion = 1 -type IndexMeta struct { +type Meta struct { path string SchemaVersion int } -func createMeta(path string) (*IndexMeta, error) { +func createMeta(path string) (*Meta, error) { exists, err := file.Exists(path) if err != nil { return nil, errors.WithMessage(err, "could not check for existence of index metadata") @@ -25,18 +25,18 @@ func createMeta(path string) (*IndexMeta, error) { return nil, errors.New("index metadata already exists") } - return &IndexMeta{ + return &Meta{ path: path, SchemaVersion: CurrentSchemaVersion, }, nil } -func openMeta(path string) (*IndexMeta, error) { +func openMeta(path string) (*Meta, error) { j, err := os.ReadFile(path) if err != nil { return nil, errors.WithMessage(err, "could not open index metadata file") } - var meta IndexMeta + var meta Meta err = json.Unmarshal(j, &meta) if err != nil { return nil, errors.WithMessage(err, "index metadata is corrupt, try replacing the index") @@ -47,7 +47,7 @@ func openMeta(path string) (*IndexMeta, error) { return &meta, nil } -func (i *IndexMeta) checkSchemaVersion() { +func (i *Meta) checkSchemaVersion() { if i.SchemaVersion < CurrentSchemaVersion { slog.Warn( "Index schema version out of date, suggest re-indexing", @@ -59,7 +59,7 @@ func (i *IndexMeta) checkSchemaVersion() { } } -func (i *IndexMeta) Save() error { +func (i *Meta) Save() error { j, err := json.Marshal(i) if err != nil { return errors.WithMessage(err, "could not prepare index metadata for saving") diff --git a/internal/search/indexer.go b/internal/index/indexer.go index a74189e..63cf1a6 100644 --- a/internal/search/indexer.go +++ b/internal/index/indexer.go @@ -1,4 +1,4 @@ -package search +package index import ( "bytes" @@ -22,13 +22,13 @@ import ( "github.com/blevesearch/bleve/v2/analysis/tokenizer/single" "github.com/blevesearch/bleve/v2/document" "github.com/blevesearch/bleve/v2/mapping" - index "github.com/blevesearch/bleve_index_api" + indexAPI "github.com/blevesearch/bleve_index_api" "github.com/pkg/errors" ) type WriteIndex struct { index bleve.Index - meta *IndexMeta + meta *Meta } func createIndexMapping() (mapping.IndexMapping, error) { @@ -161,7 +161,7 @@ func deleteIndex(dataRoot string) error { return nil } -func NewIndexer(dataRoot string, force bool) (*WriteIndex, error) { +func OpenOrCreate(dataRoot string, force bool) (*ReadIndex, *WriteIndex, bool, error) { var err error bleve.SetLog(log.Default()) @@ -170,7 +170,7 @@ func NewIndexer(dataRoot string, force bool) (*WriteIndex, error) { exists, err := file.Exists(indexPath) if err != nil { - return nil, errors.WithMessagef( + return nil, nil, exists, errors.WithMessagef( err, "could not check if index exists at path %s", indexPath, @@ -178,45 +178,51 @@ func NewIndexer(dataRoot string, force bool) (*WriteIndex, error) { } var idx bleve.Index - var meta *IndexMeta + var meta *Meta if !exists || force { if force { err = deleteIndex(dataRoot) if err != nil { - return nil, err + return nil, nil, exists, err } } idx, err = createIndex(indexPath) if err != nil { - return nil, err + return nil, nil, exists, err } meta, err = createMeta(metaPath) if err != nil { - return nil, err + return nil, nil, exists, err } err = meta.Save() if err != nil { - return nil, err + return nil, nil, exists, err } } else { idx, err = bleve.Open(indexPath) if err != nil { - return nil, errors.WithMessagef(err, "could not open index at path %s", indexPath) + return nil, nil, exists, errors.WithMessagef(err, "could not open index at path %s", indexPath) } meta, err = openMeta(metaPath) if err != nil { - return nil, err + return nil, nil, exists, err } } - return &WriteIndex{ - idx, - meta, - }, nil + return &ReadIndex{ + idx, + meta, + }, + &WriteIndex{ + idx, + meta, + }, + exists, + nil } func (i *WriteIndex) ImportOptions( @@ -257,7 +263,7 @@ func (i *WriteIndex) ImportOptions( continue } - field := document.NewTextFieldWithIndexingOptions("_data", nil, data.Bytes(), index.StoreField) + field := document.NewTextFieldWithIndexingOptions("_data", nil, data.Bytes(), indexAPI.StoreField) newDoc := doc.AddField(field) // slog.Debug("adding option to index", "name", opt.Name) diff --git a/internal/search/search.go b/internal/index/search.go index c930f15..d069510 100644 --- a/internal/search/search.go +++ b/internal/index/search.go @@ -1,10 +1,9 @@ -package search +package index import ( "bytes" "context" "encoding/gob" - "path" "searchix/internal/options" "github.com/blevesearch/bleve/v2" @@ -26,27 +25,7 @@ type Result struct { type ReadIndex struct { index bleve.Index - meta *IndexMeta -} - -func Open(dataRoot string) (*ReadIndex, error) { - indexPath := path.Join(dataRoot, indexBaseName) - metaPath := path.Join(dataRoot, metaBaseName) - - idx, err := bleve.Open(indexPath) - if err != nil { - return nil, errors.WithMessagef(err, "unable to open index at path %s", indexPath) - } - - meta, err := openMeta(metaPath) - if err != nil { - return nil, errors.WithMessagef(err, "unable to open metadata at path %s", metaPath) - } - - return &ReadIndex{ - idx, - meta, - }, nil + meta *Meta } func (index *ReadIndex) GetSource(ctx context.Context, name string) (*bleve.SearchResult, error) { diff --git a/internal/server/mux.go b/internal/server/mux.go index 9d3b29a..582d154 100644 --- a/internal/server/mux.go +++ b/internal/server/mux.go @@ -16,8 +16,8 @@ import ( "searchix/frontend" "searchix/internal/config" + search "searchix/internal/index" "searchix/internal/options" - "searchix/internal/search" "github.com/blevesearch/bleve/v2" "github.com/getsentry/sentry-go" @@ -63,15 +63,12 @@ func applyDevModeOverrides(config *config.Config) { ) } -func NewMux(config *config.Config, liveReload bool) (*http.ServeMux, error) { - slog.Debug("loading index") - index, err := search.Open(config.DataPath) - slog.Debug("loaded index") - if err != nil { - log.Fatalf("could not open search index, error: %#v", err) - } - - err = sentry.Init(sentry.ClientOptions{ +func NewMux( + config *config.Config, + index *search.ReadIndex, + liveReload bool, +) (*http.ServeMux, error) { + err := sentry.Init(sentry.ClientOptions{ EnableTracing: true, TracesSampleRate: 1.0, Dsn: config.Web.SentryDSN, diff --git a/internal/server/server.go b/internal/server/server.go index 77163d3..bb0a6ad 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "searchix/internal/config" + "searchix/internal/index" "time" "github.com/pkg/errors" @@ -16,8 +17,8 @@ type Server struct { *http.Server } -func New(conf *config.Config, liveReload bool) (*Server, error) { - mux, err := NewMux(conf, liveReload) +func New(conf *config.Config, index *index.ReadIndex, liveReload bool) (*Server, error) { + mux, err := NewMux(conf, index, liveReload) if err != nil { return nil, err } |