From 7d9e98a0996eca0372e38cee4b8826d84a9ace2b Mon Sep 17 00:00:00 2001 From: Alan Pearce Date: Sun, 26 Jan 2025 11:52:12 +0100 Subject: refactor outputs->storage for generalisation --- internal/builder/builder.go | 30 +++--- internal/outputs/files.go | 212 --------------------------------------- internal/storage/files/writer.go | 212 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 227 insertions(+), 227 deletions(-) delete mode 100644 internal/outputs/files.go create mode 100644 internal/storage/files/writer.go (limited to 'internal') diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 6e5cfc9..68eec14 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -9,8 +9,8 @@ import ( "go.alanpearce.eu/website/internal/config" "go.alanpearce.eu/website/internal/content" - "go.alanpearce.eu/website/internal/outputs" "go.alanpearce.eu/website/internal/sitemap" + "go.alanpearce.eu/website/internal/storage/files" "go.alanpearce.eu/website/templates" "go.alanpearce.eu/x/log" @@ -35,8 +35,8 @@ func joinSourcePath(src string) func(string) string { } func build(ioConfig *IOConfig, config *config.Config, log *log.Logger) (*Result, error) { - output := outputs.NewFilesOutput(ioConfig.Destination, log, &outputs.Options{ - CompressFiles: !ioConfig.Development, + storage := files.NewWriter(ioConfig.Destination, log, &files.Options{ + Compress: !ioConfig.Development, }) joinSource := joinSourcePath(ioConfig.Source) @@ -46,7 +46,7 @@ func build(ioConfig *IOConfig, config *config.Config, log *log.Logger) (*Result, Hashes: make([]string, 0), } - err := output.CopyRecursive(joinSource("static")) + err := storage.CopyRecursive(joinSource("static")) if err != nil { return nil, errors.WithMessage(err, "could not copy static files") } @@ -69,13 +69,13 @@ func build(ioConfig *IOConfig, config *config.Config, log *log.Logger) (*Result, for _, post := range posts { log.Debug("rendering post", "post", post.Basename) sitemap.AddPath(post.URL, post.Date) - if err := output.RenderToFile(templates.PostPage(config, post), post.Output); err != nil { + if err := storage.RenderToFile(templates.PostPage(config, post), post.Output); err != nil { return nil, err } } log.Debug("rendering tags list") - if err := output.RenderToFile( + if err := storage.RenderToFile( templates.TagsPage(config, "tags", mapset.Sorted(tags), "/tags"), path.Join("tags", "index.html"), ); err != nil { @@ -92,7 +92,7 @@ func build(ioConfig *IOConfig, config *config.Config, log *log.Logger) (*Result, } log.Debug("rendering tags page", "tag", tag) url := "/tags/" + tag - if err := output.RenderToFile( + if err := storage.RenderToFile( templates.TagPage(config, tag, matchingPosts, url), path.Join("tags", tag, "index.html"), ); err != nil { @@ -110,13 +110,13 @@ func build(ioConfig *IOConfig, config *config.Config, log *log.Logger) (*Result, if err != nil { return nil, errors.WithMessage(err, "could not render tag feed page") } - if err := output.WriterToFile(feed, path.Join("tags", tag, "atom.xml")); err != nil { + if err := storage.WriterToFile(feed, path.Join("tags", tag, "atom.xml")); err != nil { return nil, err } } log.Debug("rendering list page") - if err := output.RenderToFile( + if err := storage.RenderToFile( templates.ListPage(config, posts, "/post"), path.Join("post", "index.html"), ); err != nil { @@ -129,7 +129,7 @@ func build(ioConfig *IOConfig, config *config.Config, log *log.Logger) (*Result, if err != nil { return nil, errors.WithMessage(err, "could not render feed") } - if err := output.WriterToFile(feed, "atom.xml"); err != nil { + if err := storage.WriterToFile(feed, "atom.xml"); err != nil { return nil, err } @@ -138,7 +138,7 @@ func build(ioConfig *IOConfig, config *config.Config, log *log.Logger) (*Result, if err != nil { return nil, errors.WithMessage(err, "could not render feed styles") } - if err := output.OutputToFile(feedStyles, "feed-styles.xsl"); err != nil { + if err := storage.OutputToFile(feedStyles, "feed-styles.xsl"); err != nil { return nil, err } _, err = feedStyles.Seek(0, 0) @@ -160,14 +160,14 @@ func build(ioConfig *IOConfig, config *config.Config, log *log.Logger) (*Result, if err != nil { return nil, err } - if err := output.RenderToFile(templates.Homepage(config, posts, content), "index.html"); err != nil { + if err := storage.RenderToFile(templates.Homepage(config, posts, content), "index.html"); err != nil { return nil, err } // it would be nice to set LastMod here, but using the latest post // date would be wrong as the homepage has its own content file // without a date, which could be newer sitemap.AddPath("/", time.Time{}) - f, err := output.Open("index.html") + f, err := storage.Open("index.html") if err != nil { return nil, err } @@ -176,7 +176,7 @@ func build(ioConfig *IOConfig, config *config.Config, log *log.Logger) (*Result, r.Hashes = append(r.Hashes, h) log.Debug("rendering sitemap") - if err := output.WriterToFile(sitemap, "sitemap.xml"); err != nil { + if err := storage.WriterToFile(sitemap, "sitemap.xml"); err != nil { return nil, err } @@ -185,7 +185,7 @@ func build(ioConfig *IOConfig, config *config.Config, log *log.Logger) (*Result, if err != nil { return nil, err } - if err := output.OutputToFile(rob, "robots.txt"); err != nil { + if err := storage.OutputToFile(rob, "robots.txt"); err != nil { return nil, err } diff --git a/internal/outputs/files.go b/internal/outputs/files.go deleted file mode 100644 index e8da259..0000000 --- a/internal/outputs/files.go +++ /dev/null @@ -1,212 +0,0 @@ -package outputs - -import ( - "compress/gzip" - "context" - "io" - "io/fs" - "os" - "path" - "path/filepath" - - "go.alanpearce.eu/x/log" - - "github.com/a-h/templ" - "github.com/andybalholm/brotli" - "gitlab.com/tozd/go/errors" -) - -const ( - gzipLevel = 6 - brotliLevel = 9 -) - -type FilesOutput struct { - outputDirectory string - options *Options - log *log.Logger -} - -type Options struct { - CompressFiles bool -} - -func NewFilesOutput(outputDirectory string, logger *log.Logger, opts *Options) *FilesOutput { - return &FilesOutput{ - outputDirectory: outputDirectory, - options: opts, - log: logger, - } -} - -func (f *FilesOutput) CopyRecursive(src string) error { - return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - rel, err := filepath.Rel(src, path) - if err != nil { - return err - } - if d.IsDir() { - return f.mkdirp(rel) - } - - sf, err := os.Open(path) - if err != nil { - return err - } - defer sf.Close() - df, err := f.openFileAndVariants(filepath.Join(f.outputDirectory, rel)) - if err != nil { - return err - } - defer df.Close() - if _, err := io.Copy(df, sf); err != nil { - return err - } - - return nil - }) -} - -func (f *FilesOutput) Open(filename string) (io.ReadCloser, error) { - return os.Open(filepath.Join(f.outputDirectory, filename)) -} - -func (f *FilesOutput) OutputToFile(output io.Reader, filename string) error { - fn := path.Join(f.outputDirectory, filename) - if err := f.mkdirp(filepath.Dir(filename)); err != nil { - return err - } - f.log.Debug("outputting file", "filename", fn) - file, err := f.openFileAndVariants(fn) - if err != nil { - return errors.WithMessage(err, "could not open output file") - } - defer file.Close() - - if _, err := io.Copy(file, output); err != nil { - return errors.WithMessage(err, "could not write output file") - } - - return nil -} - -func (f *FilesOutput) RenderToFile(component templ.Component, filename string) error { - fn := path.Join(f.outputDirectory, filename) - if err := f.mkdirp(filepath.Dir(filename)); err != nil { - return err - } - f.log.Debug("outputting file", "filename", filename) - file, err := f.openFileAndVariants(fn) - if err != nil { - return errors.WithMessage(err, "could not open output file") - } - defer file.Close() - - if err := component.Render(context.TODO(), file); err != nil { - return errors.WithMessage(err, "could not write output file") - } - - return nil -} - -func (f *FilesOutput) WriterToFile(writer io.WriterTo, filename string) error { - fn := path.Join(f.outputDirectory, filename) - if err := f.mkdirp(filepath.Dir(filename)); err != nil { - return err - } - f.log.Debug("outputting file", "filename", fn) - file, err := f.openFileAndVariants(fn) - if err != nil { - return errors.WithMessage(err, "could not open output file") - } - defer file.Close() - - if _, err := writer.WriteTo(file); err != nil { - return errors.WithMessage(err, "could not write output file") - } - - return nil -} - -type MultiWriteCloser struct { - writers []io.WriteCloser - multiWriter io.Writer -} - -func (mw *MultiWriteCloser) Write(p []byte) (n int, err error) { - return mw.multiWriter.Write(p) -} - -func (mw *MultiWriteCloser) Close() error { - var lastErr error - for _, w := range mw.writers { - err := w.Close() - if err != nil { - lastErr = err - } - } - - return lastErr -} - -func openFileWrite(filename string) (*os.File, error) { - return os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) -} - -func openFileGz(filename string) (*gzip.Writer, error) { - filenameGz := filename + ".gz" - f, err := openFileWrite(filenameGz) - if err != nil { - return nil, err - } - - return gzip.NewWriterLevel(f, gzipLevel) -} - -func openFileBrotli(filename string) (*brotli.Writer, error) { - filenameBrotli := filename + ".br" - f, err := openFileWrite(filenameBrotli) - if err != nil { - return nil, err - } - - return brotli.NewWriterLevel(f, brotliLevel), nil -} - -func multiOpenFile(filename string) (*MultiWriteCloser, error) { - r, err := openFileWrite(filename) - if err != nil { - return nil, err - } - gz, err := openFileGz(filename) - if err != nil { - return nil, err - } - br, err := openFileBrotli(filename) - if err != nil { - return nil, err - } - - return &MultiWriteCloser{ - writers: []io.WriteCloser{r, gz, br}, - multiWriter: io.MultiWriter(r, gz, br), - }, nil -} - -func (f *FilesOutput) openFileAndVariants(filename string) (io.WriteCloser, error) { - if f.options.CompressFiles { - return multiOpenFile(filename) - } - - return openFileWrite(filename) -} - -func (f *FilesOutput) mkdirp(dir string) error { - f.log.Debug("creating directory", "dir", dir) - err := os.MkdirAll(path.Join(f.outputDirectory, dir), 0755) - - return errors.WithMessage(err, "could not create directory") -} diff --git a/internal/storage/files/writer.go b/internal/storage/files/writer.go new file mode 100644 index 0000000..6f10fe7 --- /dev/null +++ b/internal/storage/files/writer.go @@ -0,0 +1,212 @@ +package files + +import ( + "compress/gzip" + "context" + "io" + "io/fs" + "os" + "path" + "path/filepath" + + "go.alanpearce.eu/x/log" + + "github.com/a-h/templ" + "github.com/andybalholm/brotli" + "gitlab.com/tozd/go/errors" +) + +const ( + gzipLevel = 6 + brotliLevel = 9 +) + +type Files struct { + outputDirectory string + options *Options + log *log.Logger +} + +type Options struct { + Compress bool +} + +func NewWriter(outputDirectory string, logger *log.Logger, opts *Options) *Files { + return &Files{ + outputDirectory: outputDirectory, + options: opts, + log: logger, + } +} + +func (f *Files) CopyRecursive(src string) error { + return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + if d.IsDir() { + return f.mkdirp(rel) + } + + sf, err := os.Open(path) + if err != nil { + return err + } + defer sf.Close() + df, err := f.openFileAndVariants(filepath.Join(f.outputDirectory, rel)) + if err != nil { + return err + } + defer df.Close() + if _, err := io.Copy(df, sf); err != nil { + return err + } + + return nil + }) +} + +func (f *Files) Open(filename string) (io.ReadCloser, error) { + return os.Open(filepath.Join(f.outputDirectory, filename)) +} + +func (f *Files) OutputToFile(output io.Reader, filename string) error { + fn := path.Join(f.outputDirectory, filename) + if err := f.mkdirp(filepath.Dir(filename)); err != nil { + return err + } + f.log.Debug("outputting file", "filename", fn) + file, err := f.openFileAndVariants(fn) + if err != nil { + return errors.WithMessage(err, "could not open output file") + } + defer file.Close() + + if _, err := io.Copy(file, output); err != nil { + return errors.WithMessage(err, "could not write output file") + } + + return nil +} + +func (f *Files) RenderToFile(component templ.Component, filename string) error { + fn := path.Join(f.outputDirectory, filename) + if err := f.mkdirp(filepath.Dir(filename)); err != nil { + return err + } + f.log.Debug("outputting file", "filename", filename) + file, err := f.openFileAndVariants(fn) + if err != nil { + return errors.WithMessage(err, "could not open output file") + } + defer file.Close() + + if err := component.Render(context.TODO(), file); err != nil { + return errors.WithMessage(err, "could not write output file") + } + + return nil +} + +func (f *Files) WriterToFile(writer io.WriterTo, filename string) error { + fn := path.Join(f.outputDirectory, filename) + if err := f.mkdirp(filepath.Dir(filename)); err != nil { + return err + } + f.log.Debug("outputting file", "filename", fn) + file, err := f.openFileAndVariants(fn) + if err != nil { + return errors.WithMessage(err, "could not open output file") + } + defer file.Close() + + if _, err := writer.WriteTo(file); err != nil { + return errors.WithMessage(err, "could not write output file") + } + + return nil +} + +type MultiWriteCloser struct { + writers []io.WriteCloser + multiWriter io.Writer +} + +func (mw *MultiWriteCloser) Write(p []byte) (n int, err error) { + return mw.multiWriter.Write(p) +} + +func (mw *MultiWriteCloser) Close() error { + var lastErr error + for _, w := range mw.writers { + err := w.Close() + if err != nil { + lastErr = err + } + } + + return lastErr +} + +func openFileWrite(filename string) (*os.File, error) { + return os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) +} + +func openFileGz(filename string) (*gzip.Writer, error) { + filenameGz := filename + ".gz" + f, err := openFileWrite(filenameGz) + if err != nil { + return nil, err + } + + return gzip.NewWriterLevel(f, gzipLevel) +} + +func openFileBrotli(filename string) (*brotli.Writer, error) { + filenameBrotli := filename + ".br" + f, err := openFileWrite(filenameBrotli) + if err != nil { + return nil, err + } + + return brotli.NewWriterLevel(f, brotliLevel), nil +} + +func multiOpenFile(filename string) (*MultiWriteCloser, error) { + r, err := openFileWrite(filename) + if err != nil { + return nil, err + } + gz, err := openFileGz(filename) + if err != nil { + return nil, err + } + br, err := openFileBrotli(filename) + if err != nil { + return nil, err + } + + return &MultiWriteCloser{ + writers: []io.WriteCloser{r, gz, br}, + multiWriter: io.MultiWriter(r, gz, br), + }, nil +} + +func (f *Files) openFileAndVariants(filename string) (io.WriteCloser, error) { + if f.options.Compress { + return multiOpenFile(filename) + } + + return openFileWrite(filename) +} + +func (f *Files) mkdirp(dir string) error { + f.log.Debug("creating directory", "dir", dir) + err := os.MkdirAll(path.Join(f.outputDirectory, dir), 0755) + + return errors.WithMessage(err, "could not create directory") +} -- cgit 1.4.1