use redis notifications to update website
13 files changed, 586 insertions(+), 147 deletions(-)
M .gitignore → .gitignore
@@ -33,3 +33,6 @@ *_templ.go *_templ.txt /.env /site.db +/website/ +/build +/homestead
M cmd/build/main.go → cmd/build/main.go
@@ -4,9 +4,11 @@ import ( "database/sql" "fmt" "os" + "path/filepath" "go.alanpearce.eu/homestead/internal/builder" "go.alanpearce.eu/homestead/internal/config" + "go.alanpearce.eu/homestead/internal/file" "go.alanpearce.eu/homestead/internal/storage" "go.alanpearce.eu/homestead/internal/storage/files" "go.alanpearce.eu/homestead/internal/storage/sqlite"@@ -61,18 +63,23 @@ stat, err = os.Stat(options.Destination) if err != nil && !errors.Is(err, os.ErrNotExist) { panic("could not stat destination: " + err.Error()) } - if stat != nil { - if stat.IsDir() { - panic("destination is a directory") + switch { + case stat == nil: + err = os.MkdirAll(options.Destination, 0o0755) + if err != nil { + panic("could not make directory: " + err.Error()) } - err = os.Remove(options.Destination) + case !stat.IsDir(): + panic("destination is not a directory") + default: + err := file.CleanDir(options.Destination) if err != nil { - panic("could not remove destination: " + err.Error()) + panic("could not clean destination: " + err.Error()) } } var db *sql.DB - db, err = sqlite.OpenDB(options.Destination) + db, err = sqlite.OpenDB(filepath.Join(options.Destination, "site.db")) if err != nil { panic("could not open database: " + err.Error()) }@@ -82,6 +89,13 @@ Compress: options.Compress, }) if err != nil { panic("could not create storage: " + err.Error()) + } + err = file.Copy( + filepath.Join(options.Source, "config.toml"), + filepath.Join(options.Destination, "config.toml"), + ) + if err != nil { + panic("could not copy configuration: " + err.Error()) } default: panic("unknown storage type: " + options.Writer)
M fly.toml → fly.toml
@@ -9,10 +9,12 @@ [env] SERVER_PORT = "8080" SERVER_LISTEN_ADDRESS = "::" -WEBSITE_SOURCE = "/data/website" -WEBSITE_DESTINATION = "/data/public" -WEBSITE_VCS_REMOTE_URL = "https://git.alanpearce.eu/website.git" +WEBSITE_DATA_ROOT = "/data" +WEBSITE_ROOT = "/data/website" GOMEMLIMIT = "200MiB" +REDIS_ADDRESS = "redis.alanpearce.eu:6379" +REDIS_TLS_ENABLED = "true" +REDIS_TLS_INSECURE = "false" [[services]] internal_port = 8080
M go.mod → go.mod
@@ -17,14 +17,17 @@ github.com/deckarep/golang-set/v2 v2.6.0 github.com/fatih/structtag v1.2.0 github.com/fsnotify/fsnotify v1.7.0 github.com/go-git/go-git/v5 v5.12.0 + github.com/google/renameio/v2 v2.0.0 github.com/kevinpollet/nego v0.0.0-20211010160919-a65cd48cee43 github.com/klauspost/compress v1.17.11 github.com/osdevisnot/sorvor v0.4.4 + github.com/redis/go-redis/v9 v9.7.1 github.com/snabb/sitemap v1.0.4 github.com/stefanfritsch/goldmark-fences v1.0.0 github.com/yuin/goldmark v1.7.4 gitlab.com/tozd/go/errors v0.8.1 go.alanpearce.eu/x v0.0.0-20241203124832-a29434dba11a + go.uber.org/zap v1.27.0 modernc.org/sqlite v1.34.5 )@@ -40,9 +43,11 @@ github.com/benpate/domain v0.2.2 // indirect github.com/benpate/exp v0.8.3 // indirect github.com/benpate/remote v0.16.0 // indirect github.com/benpate/rosetta v0.21.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cloudflare/circl v1.3.9 // indirect github.com/cyphar/filepath-securejoin v0.2.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect@@ -68,7 +73,6 @@ github.com/sykesm/zap-logfmt v0.0.4 // indirect github.com/thessem/zap-prettyconsole v0.5.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.31.0 // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect
M go.sum → go.sum
@@ -45,7 +45,13 @@ github.com/benpate/remote v0.16.0 h1:YFXsLRjJNBPAVSZTEsgaOG7kMSHYn36z/1DmsrZU/FY= github.com/benpate/remote v0.16.0/go.mod h1:6OeZOYeEUyF0HDFaL1QPY9yboU3EvGYTNdABiRBNiF0= github.com/benpate/rosetta v0.21.2 h1:tBIfVzCv7vyLBZtF0ETAHDsvIKc28hCVBbLL7QE42pw= github.com/benpate/rosetta v0.21.2/go.mod h1:xH4gwL4OANy3PiaRq/ED4R9tU3oZ9atvH/nBzVLIfBg= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.9 h1:QFrlgFYf2Qpi8bSpVPK1HBvWpx16v/1TZivyo7pGuBE= github.com/cloudflare/circl v1.3.9/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU=@@ -59,6 +65,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=@@ -91,6 +99,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA= github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= +github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=@@ -130,6 +140,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.7.1 h1:4LhKRCIduqXqtvCUlaq9c8bdHOkICjDMrr1+Zb3osAc= +github.com/redis/go-redis/v9 v9.7.1/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
M gomod2nix.toml → gomod2nix.toml
@@ -109,6 +109,9 @@ hash = "sha256-7Gs7CS9gEYZkbu5P4hqPGBpeGZWC64VDwraSKFF+VR0=" [mod."github.com/google/pprof"] version = "v0.0.0-20240827171923-fa2c70bbbfe5" hash = "sha256-vOIHm1QA0wMX5ecg60cKT/ia/KNRy0MZPYbqFvHtFg4=" + [mod."github.com/google/renameio/v2"] + version = "v2.0.0" + hash = "sha256-8TxXyvetHewzUC9s1H5Q7HY4S1goTBLpq7f8P8g9bI8=" [mod."github.com/google/uuid"] version = "v1.6.0" hash = "sha256-VWl9sqUzdOuhW0KzQlv0gwwUQClYkmZwSydHG2sALYw="
M internal/events/file.go → internal/events/file.go
@@ -2,6 +2,7 @@ package events import ( "io/fs" + "maps" "os" "path" "path/filepath"@@ -18,6 +19,7 @@ var ( ignores = []string{ "*.templ", "*.go", + "*-journal", } checkSettleInterval = 200 * time.Millisecond )@@ -33,17 +35,19 @@ if err != nil { return nil, errors.WithMessage(err, "could not create file watcher") } + fw := &FileWatcher{ + Watcher: fsn, + log: logger, + } + for _, dir := range dirs { - err = fsn.Add(dir) + err = fw.AddRecursive(dir) if err != nil { return nil, errors.WithMessagef(err, "could not add directory %s to file watcher", dir) } } - return &FileWatcher{ - Watcher: fsn, - log: logger, - }, nil + return fw, nil } func (fw *FileWatcher) matches(name string) func(string) bool {@@ -85,53 +89,63 @@ return errors.WithMessage(err, "error walking directory tree") } -func (fw *FileWatcher) Wait(events chan<- Event, errs chan<- error) error { +func (fw *FileWatcher) GetLatestRunID() (uint64, error) { + return 0, nil +} + +func (fw *FileWatcher) Subscribe() (<-chan Event, error) { var timer *time.Timer + events := make(chan Event, 1) go func() { - var fileEvents []fsnotify.Event + fileEvents := make(map[string]fsnotify.Op) for { select { case baseEvent := <-fw.Watcher.Events: if !fw.ignored(baseEvent.Name) { - fw.log.Debug( - "watcher event", - "name", - baseEvent.Name, - "op", - baseEvent.Op.String(), - ) if baseEvent.Has(fsnotify.Create) || baseEvent.Has(fsnotify.Rename) { f, err := os.Stat(baseEvent.Name) if err != nil { - errs <- errors.WithMessagef(err, "error handling event %s", baseEvent.Op.String()) + fw.log.Warn( + "error handling event", + "op", + baseEvent.Op, + "name", + baseEvent.Name, + ) } else if f.IsDir() { err = fw.Add(baseEvent.Name) if err != nil { - errs <- errors.WithMessage(err, "error adding new folder to watcher") + fw.log.Warn("error adding new folder to watcher", "error", err) } } } if baseEvent.Has(fsnotify.Rename) || baseEvent.Has(fsnotify.Write) || baseEvent.Has(fsnotify.Create) || baseEvent.Has(fsnotify.Chmod) { - fileEvents = append(fileEvents, baseEvent) + fileEvents[baseEvent.Name] |= baseEvent.Op if timer == nil { timer = time.AfterFunc(checkSettleInterval, func() { + fw.log.Debug( + "file update event", + "changed", + slices.Collect(maps.Keys(fileEvents)), + ) events <- Event{ - FileEvents: fileEvents, - Revision: "", + FSEvent: FSEvent{ + Events: fileEvents, + }, } - fileEvents = []fsnotify.Event{} + clear(fileEvents) }) } timer.Reset(checkSettleInterval) } } case err := <-fw.Watcher.Errors: - errs <- errors.WithMessage(err, "error in watcher") + fw.log.Warn("error in watcher", "error", err) } } }() - return nil + return events, nil }
A internal/events/redis.go
@@ -0,0 +1,126 @@ +package events + +import ( + "context" + "crypto/tls" + "fmt" + "strconv" + "time" + + "github.com/redis/go-redis/v9" + "gitlab.com/tozd/go/errors" + "go.alanpearce.eu/x/log" + "go.uber.org/zap" +) + +const db = 0 +const key = "run_id" +const fallbackRunID uint64 = 210 + +type RedisOptions struct { + Enabled bool `conf:"default:false"` + Address string `conf:"default:localhost:6379"` + Username string `conf:"default:default"` + Password string `conf:"default:default"` + TLSEnabled bool `conf:"default:false"` +} + +type Logger struct { + log *zap.SugaredLogger +} + +func (l *Logger) Printf(_ context.Context, format string, v ...interface{}) { + l.log.Infof(format, v...) +} + +type RedisListener struct { + client *redis.Client + log *log.Logger +} + +func NewRedisListener(opts *RedisOptions, log *log.Logger) (*RedisListener, error) { + clientConfig := &redis.Options{ + Addr: opts.Address, + Username: opts.Username, + Password: opts.Password, + } + if opts.TLSEnabled { + clientConfig.TLSConfig = &tls.Config{ + InsecureSkipVerify: false, + MinVersion: tls.VersionTLS13, + } + } + redis.SetLogger(&Logger{log: log.GetLogger().Sugar()}) + client := redis.NewClient(clientConfig) + + log.Debug( + "connecting to redis", + "address", + opts.Address, + "username", + opts.Username, + "tls", + opts.TLSEnabled, + ) + + return &RedisListener{ + client: client, + log: log, + }, nil +} + +func (rl *RedisListener) GetLatestRunID() (uint64, error) { + ctx, cancel := context.WithTimeout(context.TODO(), 2*time.Second) + defer cancel() + + payload, err := rl.client.Get(ctx, key).Result() + if err != nil { + return fallbackRunID, errors.WithMessage(err, "could not get latest run ID") + } + + runID, err := strconv.ParseUint(payload, 10, 64) + if err != nil { + return fallbackRunID, errors.WithMessage(err, "could not parse latest run ID") + } + + rl.log.Debug("redis response", "payload", payload, "run_id", runID) + + return runID, nil +} + +// requires `redis-cli config set notify-keyspace-events KEA` +func getKeyspaceName(key string) string { + return fmt.Sprintf("__keyspace@%d__:%s", db, key) +} + +func (rl *RedisListener) Subscribe() (<-chan Event, error) { + events := make(chan Event, 1) + ctx := context.TODO() + channel := getKeyspaceName(key) + pubsub := rl.client.Subscribe(ctx, channel) + rl.log.Debug("subscribing", "channel", channel) + + _, err := pubsub.Receive(ctx) + if err != nil { + return nil, errors.WithMessage(err, "could not subscribe to channel") + } + + go func(ch <-chan *redis.Message) { + for msg := range ch { + rl.log.Debug("got event", "payload", msg.Payload) + + runID, err := rl.GetLatestRunID() + if err != nil { + rl.log.Warn("could not get latest run ID") + } + + events <- Event{ + CIEvent: CIEvent{ + RunID: runID, + }, + } + } + }(pubsub.Channel()) + + return events, nil +}
M internal/events/update.go → internal/events/update.go
@@ -2,11 +2,20 @@ package events import "github.com/fsnotify/fsnotify" -type Listener interface { - Wait(chan<- Event, chan<- error) error +type CIEvent struct { + RunID uint64 +} + +type FSEvent struct { + Events map[string]fsnotify.Op } type Event struct { - FileEvents []fsnotify.Event - Revision string + CIEvent + FSEvent +} + +type Listener interface { + GetLatestRunID() (uint64, error) + Subscribe() (<-chan Event, error) }
A internal/fetcher/fetcher.go
@@ -0,0 +1,218 @@ +package fetcher + +import ( + "context" + "io" + "io/fs" + "net/http" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "github.com/google/renameio/v2" + "gitlab.com/tozd/go/errors" + "go.alanpearce.eu/homestead/internal/config" + "go.alanpearce.eu/homestead/internal/events" + "go.alanpearce.eu/x/log" +) + +var files = []string{"config.toml", "site.db"} +var numericFilename = regexp.MustCompile("[0-9]{3,}") +var timeout = 10 * time.Second + +type Fetcher struct { + options *Options + log *log.Logger + updater events.Listener +} + +type Options struct { + Root string + RedisEnabled bool + FetchURL config.URL + Listener events.Listener +} + +func New(log *log.Logger, options *Options) Fetcher { + return Fetcher{ + log: log, + options: options, + updater: options.Listener, + } +} + +func (f *Fetcher) getArtefacts(run uint64) error { + runID := strconv.FormatUint(run, 10) + f.log.Debug("getting artefacts", "run_id", runID) + + err := os.MkdirAll(filepath.Join(f.options.Root, runID), 0755) + if err != nil { + return errors.WithMessage(err, "could not create directory") + } + + for _, file := range files { + err := f.getFile(runID, file) + if err != nil { + return errors.WithMessage(err, "could not fetch file") + } + } + + err = renameio.Symlink(runID, filepath.Join(f.options.Root, "current")) + if err != nil { + return errors.WithMessage(err, "could not create/update symlink") + } + + return nil +} + +func (f *Fetcher) checkFolder() error { + contents, err := os.ReadDir(f.options.Root) + if err != nil { + return errors.WithMessage(err, "could not read root directory") + } + var badFiles []string + for _, f := range contents { + name := f.Name() + if !(name == "current" || numericFilename.MatchString(name)) { + badFiles = append(badFiles, name) + } + } + + if len(badFiles) > 0 { + return errors.Basef("unexpected files in root directory: %s", strings.Join(badFiles, ", ")) + } + + return nil +} + +func (f *Fetcher) getFile(runID, basename string) error { + filename := filepath.Join(f.options.Root, runID, basename) + url := f.options.FetchURL.JoinPath(runID, basename).String() + + f.log.Debug("getting file", "filename", filename, "url", url) + + file, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return errors.WithMessage(err, "could not open file") + } + defer file.Close() + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return errors.WithMessage(err, "could not create request") + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return errors.WithMessage(err, "could not issue request") + } + + _, err = io.Copy(file, res.Body) + if err != nil { + return errors.WithMessage(err, "could not write file") + } + + err = file.Sync() + if err != nil { + return errors.WithMessage(err, "could not sync file") + } + + return nil +} + +func (f *Fetcher) getCurrentVersion() (uint64, error) { + target, err := os.Readlink(filepath.Join(f.options.Root, "current")) + if err != nil && errors.Is(err, fs.ErrNotExist) { + return 0, errors.WithMessage(err, "could not stat current link") + } + runID, err := strconv.ParseUint(target, 10, 64) + if err != nil { + return 0, errors.WithMessagef(err, "unexpected symlink target (current -> %s)", target) + } + + return runID, nil +} + +func (f *Fetcher) initialiseStorage() (uint64, error) { + latest, err := f.updater.GetLatestRunID() + if err != nil { + f.log.Warn("could not get latest run ID, using fallback", "error", err) + } + + current, err := f.getCurrentVersion() + if err != nil { + f.log.Warn("could not get current version", "error", err) + } + + f.log.Debug("versions", "current", current, "latest", latest) + + if latest > current { + err = f.getArtefacts(latest) + if err != nil { + return latest, errors.WithMessage(err, "could not fetch artefacts") + } + + return latest, nil + } + + return current, nil +} + +func (f *Fetcher) Subscribe() (<-chan string, error) { + ch := make(chan string, 1) + err := f.checkFolder() + if err != nil { + return nil, err + } + + var root string + if f.options.RedisEnabled { + root = f.path("current") + } else { + runID, err := f.initialiseStorage() + if err != nil { + return nil, err + } + root = f.path(strconv.FormatUint(runID, 10)) + } + updates, err := f.updater.Subscribe() + if err != nil { + return nil, errors.WithMessage(err, "could not subscribe to updates") + } + + go func() { + ch <- root + + for update := range updates { + if update.RunID == 0 { + if !f.options.RedisEnabled { + f.log.Warn("got zero runID") + + continue + } + + ch <- f.path("current") + } else { + err := f.getArtefacts(update.RunID) + if err != nil { + f.log.Warn("could not get artefacts for version", "run_id", update.RunID, "error", err) + + continue + } + + ch <- f.path(strconv.FormatUint(update.RunID, 10)) + } + } + }() + + return ch, nil +} + +func (f *Fetcher) path(runID string) string { + return filepath.Join(f.options.Root, runID) +}
A internal/file/file.go
@@ -0,0 +1,62 @@ +package file + +import ( + "io" + "io/fs" + "os" + "path/filepath" + + "gitlab.com/tozd/go/errors" +) + +func Exists(path string) bool { + stat, err := os.Stat(path) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + panic("could not stat path " + path + ": " + err.Error()) + } + + return stat != nil +} + +func Copy(src, dest string) error { + stat, err := os.Stat(src) + if err != nil { + return errors.WithMessage(err, "could not stat source file") + } + + sf, err := os.Open(src) + if err != nil { + return errors.WithMessage(err, "could not open source file for reading") + } + defer sf.Close() + + df, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, stat.Mode()) + if err != nil { + return errors.WithMessage(err, "could not open destination file for writing") + } + defer df.Close() + + if _, err := io.Copy(df, sf); err != nil { + return errors.WithMessage(err, "could not copy") + } + + err = df.Sync() + if err != nil { + return errors.WithMessage(err, "could not call fsync") + } + + return nil +} + +func CleanDir(dir string) (err error) { + files, err := os.ReadDir(dir) + if err != nil { + return errors.WithMessage(err, "could not read directory") + } + + for _, file := range files { + err = errors.Join(err, os.Remove(filepath.Join(dir, file.Name()))) + } + + return err +}
M internal/website/website.go → internal/website/website.go
@@ -2,18 +2,21 @@ package website import ( "net/http" + "os" + "path/filepath" "slices" "strings" + "sync" "gitlab.com/tozd/go/errors" - "go.alanpearce.eu/homestead/internal/builder" "go.alanpearce.eu/homestead/internal/config" "go.alanpearce.eu/homestead/internal/events" + "go.alanpearce.eu/homestead/internal/fetcher" + "go.alanpearce.eu/homestead/internal/file" ihttp "go.alanpearce.eu/homestead/internal/http" "go.alanpearce.eu/homestead/internal/server" "go.alanpearce.eu/homestead/internal/storage" "go.alanpearce.eu/homestead/internal/storage/sqlite" - "go.alanpearce.eu/homestead/internal/vcs" "go.alanpearce.eu/homestead/templates" "go.alanpearce.eu/x/log"@@ -22,16 +25,14 @@ "github.com/osdevisnot/sorvor/pkg/livereload" ) type Options struct { - Source string `conf:"default:../website"` - DBPath string `conf:"default:site.db"` + DataRoot string `conf:"noprint"` + Root string `conf:"default:./website"` Redirect bool `conf:"default:true"` Development bool `conf:"default:false,flag:dev"` - BaseURL config.URL `conf:"default:localhost"` - VCS struct { - Branch string `conf:"default:main"` - RemoteURL config.URL `conf:"default:https://git.alanpearce.eu/website"` - } + FetchURL config.URL `conf:"default:https://ci.alanpearce.eu/archive/website/"` + BaseURL config.URL + Redis *events.RedisOptions LiveReload *livereload.LiveReload `conf:"-"` }@@ -54,34 +55,78 @@ App: &server.App{ Shutdown: func() {}, }, } - builderOptions := &builder.Options{ - Source: opts.Source, - Development: opts.Development, - } - repo, exists, err := vcs.CloneOrOpen(&vcs.Options{ - LocalPath: opts.Source, - RemoteURL: opts.VCS.RemoteURL, - Branch: opts.VCS.Branch, - }, log.Named("vcs")) + err := prepareRootDirectory(opts.DataRoot, opts.Root) if err != nil { - return nil, errors.WithMessage(err, "could not open repository") + return nil, errors.WithMessage(err, "could not prepare root directory") } - builderOptions.Repo = repo - if exists && !opts.Development { - _, err := repo.Update("") - if err != nil { - return nil, errors.WithMessage(err, "could not update repository") - } + var listener events.Listener + if opts.Redis.Enabled { + log.Debug("using redis listener") + listener, err = events.NewRedisListener(opts.Redis, log.Named("redis")) + } else { + log.Debug("using file watcher") + listener, err = events.NewFileWatcher(log.Named("events"), "website") + } + if err != nil { + return nil, errors.WithMessage(err, "could not create update listener") } - log.Debug("getting config from source", "source", opts.Source) - cfg, err := config.GetConfig(opts.Source, log) + var cfg *config.Config + fetcher := fetcher.New(log.Named("fetcher"), &fetcher.Options{ + FetchURL: opts.FetchURL, + RedisEnabled: false, + Root: opts.Root, + Listener: listener, + }) + roots, err := fetcher.Subscribe() if err != nil { - return nil, errors.WithMessage(err, "could not load configuration") + return nil, errors.WithMessage(err, "could not set up fetcher") } + firstUpdate := make(chan bool) + go func() { + updated := sync.OnceFunc(func() { + firstUpdate <- true + close(firstUpdate) + }) + for root := range roots { + log.Debug("getting config from source", "source", root) + cfg, err = config.GetConfig(root, log) + if err != nil { + log.Panic("could not load configuration", "error", err) + } + website.config = cfg + + if opts.Development { + cfg.CSP.ScriptSrc = slices.Insert(cfg.CSP.ScriptSrc, 0, "'unsafe-inline'") + cfg.CSP.ConnectSrc = slices.Insert(cfg.CSP.ConnectSrc, 0, "'self'") + } + + if opts.BaseURL.Hostname() != "" { + cfg.BaseURL = opts.BaseURL + } + + website.Domain = cfg.BaseURL.Hostname() + + siteDB := filepath.Join(root, "site.db") + db, err := sqlite.OpenDB(siteDB) + if err != nil { + log.Panic("could not open database", "error", err) + } + + website.reader, err = sqlite.NewReader(db, log.Named("reader")) + if err != nil { + log.Panic("could not create database reader", "error", err) + } + + updated() + } + }() + + <-firstUpdate + mux := ihttp.NewServeMux() mux.HandleError(func(err *ihttp.Error, w http.ResponseWriter, r *http.Request) { if strings.Contains(r.Header.Get("Accept"), "text/html") {@@ -95,84 +140,6 @@ http.Error(w, err.Message, err.Code) } }) - if opts.Development { - opts.DBPath = ":memory:" - - cfg.CSP.ScriptSrc = slices.Insert(cfg.CSP.ScriptSrc, 0, "'unsafe-inline'") - cfg.CSP.ConnectSrc = slices.Insert(cfg.CSP.ConnectSrc, 0, "'self'") - - cfg.BaseURL = opts.BaseURL - } - - db, err := sqlite.OpenDB(opts.DBPath) - if err != nil { - return nil, errors.WithMessage(err, "could not open database") - } - - builderOptions.Storage, err = sqlite.NewWriter(db, log.Named("storage"), &sqlite.Options{ - Compress: true, - }) - if err != nil { - return nil, errors.WithMessage(err, "could not create storage writer") - } - - website.Domain = cfg.BaseURL.Hostname() - - err = rebuild(builderOptions, cfg, log) - if err != nil { - return nil, errors.WithMessage(err, "could not build site") - } - - if opts.Development { - fw, err := events.NewFileWatcher( - log.Named("watcher"), - opts.Source, - "templates", - ) - if err != nil { - return nil, errors.WithMessage(err, "could not create file listener") - } - - go func(events chan events.Event, errs chan error) { - err := fw.Wait(events, errs) - if err != nil { - log.Panic("could not start update listener", "error", err) - } - for event := range events { - filename := event.FileEvents[0].Name - log.Info("rebuilding site", "changed_file", filename) - db.Close() - db, err = sqlite.OpenDB(opts.DBPath) - if err != nil { - log.Error("error opening database", "error", err) - } - website.reader, err = sqlite.NewReader(db, log.Named("reader")) - if err != nil { - log.Error("error creating sqlite reader", "error", err) - } - builderOptions.Storage, err = sqlite.NewWriter( - db, - log.Named("storage"), - &sqlite.Options{}, - ) - if err != nil { - log.Error("error creating sqlite writer", "error", err) - } - - err := rebuild(builderOptions, cfg, log) - if err != nil { - log.Error("error rebuilding site", "error", err) - } - opts.LiveReload.Reload() - } - }(make(chan events.Event, 1), make(chan error, 1)) - } - - website.reader, err = sqlite.NewReader(db, log.Named("reader")) - if err != nil { - return nil, errors.WithMessage(err, "error creating sqlite reader") - } - website.acctResource = "acct:" + cfg.Email website.me = digit.NewResource(website.acctResource). Link("http://openid.net/specs/connect/1.0/issuer", "", cfg.OIDCHost.String())@@ -182,16 +149,20 @@ mux.HandleFunc("/.well-known/webfinger", website.webfinger) const oidcPath = "/.well-known/openid-configuration" mux.ServeMux.Handle(oidcPath, ihttp.RedirectHandler(cfg.OIDCHost.JoinPath(oidcPath), 302)) - website.config = cfg website.App.Handler = mux return website, nil } -func rebuild(builderConfig *builder.Options, config *config.Config, log *log.Logger) error { - err := builder.BuildSite(builderConfig, config, log.Named("builder")) - if err != nil { - return errors.WithMessage(err, "could not build site") +func prepareRootDirectory(dataRoot, root string) error { + if !file.Exists(root) { + err := os.MkdirAll(root, 0755) + if err != nil { + return errors.WithMessage(err, "could not create root directory") + } + } else if dataRoot != "" && file.Exists(filepath.Join(dataRoot, "public")) { + // must be pre-sqlite if this exists + return file.CleanDir(dataRoot) } return nil