package fetcher import ( "context" "net/url" "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/index" "gitlab.com/tozd/go/errors" "go.alanpearce.eu/x/log" ) type DownloadFetcher struct { Source *config.Source SourceFile string Logger *log.Logger } func NewDownloadFetcher( source *config.Source, logger *log.Logger, ) (*DownloadFetcher, errors.E) { switch source.Importer { case config.Options: return &DownloadFetcher{ Source: source, Logger: logger, }, nil default: return nil, errors.Errorf("unsupported importer type %s", source.Importer) } } var files = map[string]string{ "revision": "revision", "options": "options.json", } func (i *DownloadFetcher) FetchIfNeeded( ctx context.Context, sourceMeta *index.SourceMeta, ) (*FetchedFiles, errors.E) { f := &FetchedFiles{} sourceUpdated := sourceMeta.Updated for key, filename := range files { fetchURL, baseErr := url.JoinPath(i.Source.URL, filename) if baseErr != nil { return nil, errors.WithMessagef( baseErr, "could not build URL with elements %s and %s", i.Source.URL, filename, ) } i.Logger.Debug("preparing to fetch URL", "url", fetchURL) body, mtime, err := fetchFileIfNeeded(ctx, i.Logger, sourceUpdated, fetchURL) if err != nil { i.Logger.Warn("failed to fetch file", "url", fetchURL, "error", err) return nil, err } // don't bother to issue requests for the later files if mtime.Before(sourceUpdated) { break } sourceMeta.Updated = mtime switch key { case "revision": f.Revision = body case "options": f.Options = body default: return f, errors.Errorf("unknown file kind %s", key) } } return f, nil }