use buffers as interface to storage

9 files changed, 316 insertions, 122 deletions

diff --git a/internal/buffer/buffer.go b/internal/buffer/buffer.go
new file mode 100644
index 0000000..055bf7f
--- /dev/null
+++ b/internal/buffer/buffer.go
@@ -0,0 +1,95 @@
+package buffer
+
+import (
+	"io"
+)
+
+type Buffer struct {
+	buf []byte
+	pos int
+	len int
+}
+
+func NewBuffer(buf []byte) *Buffer {
+	return &Buffer{
+		buf: buf,
+		pos: 0,
+		len: len(buf),
+	}
+}
+
+// Read implements io.Reader's Read method
+func (b *Buffer) Read(p []byte) (int, error) {
+	if b.pos >= b.len {
+		return 0, io.EOF
+	}
+
+	n := len(p)
+	if n > b.len-b.pos {
+		n = b.len - b.pos
+	}
+
+	copy(p[:n], b.buf[b.pos:b.pos+n])
+	b.pos += n
+
+	return n, nil
+}
+
+// Write appends the contents of p to the buffer's data.
+func (b *Buffer) Write(p []byte) (int, error) {
+	if len(b.buf) < b.len+len(p) {
+		newLen := b.len + len(p)
+		if cap(b.buf) >= newLen {
+			b.buf = b.buf[:newLen]
+		} else {
+			newBuf := make([]byte, newLen*2)
+			copy(newBuf, b.buf[:b.len])
+			b.buf = newBuf
+		}
+	}
+
+	copy(b.buf[b.len:], p)
+	b.len += len(p)
+
+	return len(p), nil
+}
+
+func (b *Buffer) Len() int {
+	return b.len
+}
+
+// Reset resets the buffer to be empty. The underlying array is reused if possible.
+func (b *Buffer) Reset() {
+	b.len = 0
+	b.pos = 0
+}
+
+// Seek moves the read position by offset bytes relative to whence (Start, Current, End)
+func (b *Buffer) Seek(offset int64, whence int) (int64, error) {
+	var newpos int
+
+	switch whence {
+	case io.SeekStart:
+		newpos = int(offset)
+	case io.SeekCurrent:
+		newpos = b.pos + int(offset)
+	case io.SeekEnd:
+		newpos = b.len + int(offset)
+	default:
+		return 0, io.EOF
+	}
+
+	if newpos < 0 {
+		newpos = 0
+	} else if newpos > b.len {
+		newpos = b.len
+	}
+
+	b.pos = newpos
+
+	return int64(newpos), nil
+}
+
+func (b *Buffer) Bytes() []byte {
+	return b.buf[:b.len]
+}
diff --git a/internal/buffer/buffer_test.go b/internal/buffer/buffer_test.go
new file mode 100644
index 0000000..9cddeca
--- /dev/null
+++ b/internal/buffer/buffer_test.go
@@ -0,0 +1,88 @@
+package buffer
+
+import (
+	"io"
+	"testing"
+)
+
+func TestWrite(t *testing.T) {
+	b := Buffer{}
+	data := []byte("test")
+
+	n, err := b.Write(data)
+	if n != len(data) || err != nil {
+		t.Errorf("Write failed: expected %d bytes, got %d, error: %v", len(data), n, err)
+	}
+
+	if b.Len() != len(data) {
+		t.Errorf("Len is incorrect after write: expected %d, got %d", len(data), b.Len())
+	}
+
+	if string(b.Bytes()) != "test" {
+		t.Error("Bytes returned after write do not match written data")
+	}
+}
+
+func TestRead(t *testing.T) {
+	b := NewBuffer([]byte("testdata"))
+	p := make([]byte, 3)
+	n, err := b.Read(p)
+
+	if n != 3 || string(p[:n]) != "tes" {
+		t.Errorf("Read returned incorrect data: expected 'tes', got '%s'", p[:n])
+	}
+
+	b.Reset()
+	b.Write([]byte("abc"))
+	p = make([]byte, 2)
+	n, err = b.Read(p)
+
+	if n != 2 || string(p) != "ab" {
+		t.Errorf("Read after reset failed: expected 'ab', got '%s'", p)
+	}
+
+	b.pos = b.len
+	n, err = b.Read(p)
+	if n != 0 || err != io.EOF {
+		t.Errorf("Reading beyond buffer did not return EOF: n=%d, err=%v", n, err)
+	}
+}
+
+func TestReset(t *testing.T) {
+	b := NewBuffer([]byte("test"))
+	b.Write([]byte("data"))
+
+	if b.Len() != 8 || b.pos != 0 {
+		t.Errorf("Initial buffer state incorrect: len=%d, pos=%d", b.Len(), b.pos)
+	}
+
+	b.Reset()
+	if b.Len() != 0 || b.pos != 0 {
+		t.Errorf("Reset did not clear buffer correctly: len=%d, pos=%d", b.Len(), b.pos)
+	}
+}
+
+func TestSeek(t *testing.T) {
+	b := NewBuffer([]byte("test"))
+	tests := []struct {
+		offset int64
+		whence int
+		expect int64
+		err    error
+	}{
+		{2, io.SeekStart, 2, nil},
+		{-1, io.SeekCurrent, 1, nil},
+		{-2, io.SeekEnd, int64(len("test") - 2), nil},
+		{5, io.SeekStart, int64(len("test")), nil},
+		{-10, io.SeekEnd, 0, nil},
+		{0, 999, 0, io.EOF}, // Invalid whence test
+	}
+
+	for _, tt := range tests {
+		pos, err := b.Seek(tt.offset, tt.whence)
+		if pos != tt.expect || (err != tt.err && !((err == nil) && (tt.err == nil))) {
+			t.Errorf("Seek(%d, %d): expected %d with error %v, got %d and %v",
+				tt.offset, tt.whence, tt.expect, tt.err, pos, err)
+		}
+	}
+}
diff --git a/internal/builder/builder.go b/internal/builder/builder.go
index 8d2f0a4..68f970f 100644
--- a/internal/builder/builder.go
+++ b/internal/builder/builder.go
@@ -1,6 +1,7 @@
 package builder
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"io/fs"
@@ -10,6 +11,7 @@ import (
 	"slices"
 	"time"
 
+	"go.alanpearce.eu/website/internal/buffer"
 	"go.alanpearce.eu/website/internal/config"
 	"go.alanpearce.eu/website/internal/content"
 	"go.alanpearce.eu/website/internal/sitemap"
@@ -39,6 +41,7 @@ func joinSourcePath(src string) func(string) string {
 }
 
 func copyRecursive(storage storage.Writer, src string) error {
+	buf := new(buffer.Buffer)
 	return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
@@ -56,12 +59,11 @@ func copyRecursive(storage storage.Writer, src string) error {
 			return err
 		}
 		defer sf.Close()
-		df, err := storage.OpenFileAndVariants(rel)
-		if err != nil {
+		buf.Reset()
+		if _, err := io.Copy(buf, sf); err != nil {
 			return err
 		}
-		defer df.Close()
-		if _, err := io.Copy(df, sf); err != nil {
+		if err := storage.Write("/"+rel, buf); err != nil {
 			return err
 		}
 
@@ -76,6 +78,8 @@ func build(
 	config *config.Config,
 	log *log.Logger,
 ) (*Result, error) {
+	ctx := context.TODO()
+	buf := new(buffer.Buffer)
 	joinSource := joinSourcePath(ioConfig.Source)
 
 	log.Debug("output", "dir", ioConfig.Destination)
@@ -106,16 +110,21 @@ func build(
 	for _, post := range posts {
 		log.Debug("rendering post", "post", post.Basename)
 		sitemap.AddPath(post.URL, post.Date)
-		if err := storage.RenderToFile(templates.PostPage(config, post), post.Output); err != nil {
+		if err := templates.PostPage(config, post).Render(ctx, buf); err != nil {
+			return nil, errors.WithMessage(err, "could not render post")
+		}
+
+		if err := storage.Write(post.URL, buf); err != nil {
 			return nil, err
 		}
 	}
 
 	log.Debug("rendering tags list")
-	if err := storage.RenderToFile(
-		templates.TagsPage(config, "tags", mapset.Sorted(tags), "/tags"),
-		"tags/",
-	); err != nil {
+	buf.Reset()
+	if err := templates.TagsPage(config, "tags", mapset.Sorted(tags), "/tags").Render(ctx, buf); err != nil {
+		return nil, err
+	}
+	if err := storage.Write("/tags/", buf); err != nil {
 		return nil, err
 	}
 	sitemap.AddPath("/tags/", lastMod)
@@ -128,11 +137,12 @@ func build(
 			}
 		}
 		log.Debug("rendering tags page", "tag", tag)
-		url := path.Join("tags", tag) + "/"
-		if err := storage.RenderToFile(
-			templates.TagPage(config, tag, matchingPosts, url),
-			url,
-		); err != nil {
+		url := path.Join("/tags", tag) + "/"
+		buf.Reset()
+		if err := templates.TagPage(config, tag, matchingPosts, url).Render(ctx, buf); err != nil {
+			return nil, err
+		}
+		if err = storage.Write(url, buf); err != nil {
 			return nil, err
 		}
 		sitemap.AddPath(url, matchingPosts[0].Date)
@@ -147,16 +157,21 @@ func build(
 		if err != nil {
 			return nil, errors.WithMessage(err, "could not render tag feed page")
 		}
-		if err := storage.WriterToFile(feed, path.Join("tags", tag, "atom.xml")); err != nil {
+		buf.Reset()
+		if _, err := feed.WriteTo(buf); err != nil {
+			return nil, err
+		}
+		if err := storage.Write(path.Join("/tags", tag, "atom.xml"), buf); err != nil {
 			return nil, err
 		}
 	}
 
 	log.Debug("rendering list page")
-	if err := storage.RenderToFile(
-		templates.ListPage(config, posts, "/post"),
-		"post/",
-	); err != nil {
+	buf.Reset()
+	if err := templates.ListPage(config, posts, "/post").Render(ctx, buf); err != nil {
+		return nil, err
+	}
+	if err := storage.Write("/post/", buf); err != nil {
 		return nil, err
 	}
 	sitemap.AddPath("/post/", lastMod)
@@ -166,7 +181,11 @@ func build(
 	if err != nil {
 		return nil, errors.WithMessage(err, "could not render feed")
 	}
-	if err := storage.WriterToFile(feed, "atom.xml"); err != nil {
+	buf.Reset()
+	if _, err := feed.WriteTo(buf); err != nil {
+		return nil, err
+	}
+	if err := storage.Write("/atom.xml", buf); err != nil {
 		return nil, err
 	}
 
@@ -175,7 +194,11 @@ func build(
 	if err != nil {
 		return nil, errors.WithMessage(err, "could not render feed styles")
 	}
-	if err := storage.OutputToFile(feedStyles, "feed-styles.xsl"); err != nil {
+	buf.Reset()
+	if _, err := feedStyles.WriteTo(buf); err != nil {
+		return nil, err
+	}
+	if err := storage.Write("/feed-styles.xsl", buf); err != nil {
 		return nil, err
 	}
 	_, err = feedStyles.Seek(0, 0)
@@ -197,23 +220,25 @@ func build(
 	if err != nil {
 		return nil, err
 	}
-	if err := storage.RenderToFile(templates.Homepage(config, posts, content), "/"); err != nil {
+	buf.Reset()
+	if err := templates.Homepage(config, posts, content).Render(ctx, buf); err != nil {
+		return nil, err
+	}
+	if err := storage.Write("/", buf); err != nil {
 		return nil, err
 	}
+
 	// it would be nice to set LastMod here, but using the latest post
 	// date would be wrong as the homepage has its own content file
 	// without a date, which could be newer
 	sitemap.AddPath("/", time.Time{})
-	f, err := storage.Open("/")
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-	h, _ = getHTMLStyleHash(f)
+	h, _ = getHTMLStyleHash(buf)
 	r.Hashes = append(r.Hashes, h)
 
 	log.Debug("rendering sitemap")
-	if err := storage.WriterToFile(sitemap, "sitemap.xml"); err != nil {
+	buf.Reset()
+	sitemap.WriteTo(buf)
+	if err := storage.Write("/sitemap.xml", buf); err != nil {
 		return nil, err
 	}
 
@@ -222,7 +247,11 @@ func build(
 	if err != nil {
 		return nil, err
 	}
-	if err := storage.OutputToFile(rob, "robots.txt"); err != nil {
+	buf.Reset()
+	if _, err := io.Copy(buf, rob); err != nil {
+		return nil, err
+	}
+	if err := storage.Write("/robots.txt", buf); err != nil {
 		return nil, err
 	}
 
@@ -238,10 +267,12 @@ func BuildSite(ioConfig *IOConfig, cfg *config.Config, log *log.Logger) (*Result
 	templates.Setup()
 	loadCSS(ioConfig.Source)
 
-	var storage storage.Writer
-	storage = files.NewWriter(ioConfig.Destination, log, &files.Options{
+	storage, err := files.NewWriter(ioConfig.Destination, log, &files.Options{
 		Compress: !ioConfig.Development,
 	})
+	if err != nil {
+		return nil, errors.WithMessage(err, "could not create storage writer")
+	}
 
 	return build(storage, ioConfig, cfg, log)
 }
diff --git a/internal/storage/file.go b/internal/storage/file.go
index f588bf3..38824b6 100644
--- a/internal/storage/file.go
+++ b/internal/storage/file.go
@@ -1,8 +1,9 @@
 package storage
 
 import (
-	"io"
 	"time"
+
+	"go.alanpearce.eu/website/internal/buffer"
 )
 
 type File struct {
@@ -10,7 +11,7 @@ type File struct {
 	ContentType  string
 	LastModified time.Time
 	Etag         string
-	Encodings    map[string]io.ReadSeekCloser
+	Encodings    map[string]*buffer.Buffer
 }
 
 func (f *File) AvailableEncodings() []string {
diff --git a/internal/storage/files/file.go b/internal/storage/files/file.go
index a71811c..b79c43c 100644
--- a/internal/storage/files/file.go
+++ b/internal/storage/files/file.go
@@ -10,19 +10,18 @@ import (
 	"strings"
 
 	"gitlab.com/tozd/go/errors"
+	"go.alanpearce.eu/website/internal/buffer"
 	"go.alanpearce.eu/website/internal/storage"
 )
 
-type File struct {
-	storage.File
-}
+type Content io.ReadSeekCloser
 
 var encodings = map[string]string{
 	"br":   ".br",
 	"gzip": ".gz",
 }
 
-func (r *Reader) OpenFile(path string, filename string) (*File, error) {
+func (r *Reader) OpenFile(path string, filename string) (*storage.File, error) {
 	f, err := os.Open(filename)
 	if err != nil {
 		return nil, errors.WithMessage(err, "could not open file for reading")
@@ -37,15 +36,18 @@ func (r *Reader) OpenFile(path string, filename string) (*File, error) {
 		return nil, errors.WithMessage(err, "could not calculate etag")
 	}
 
-	file := &File{
-		File: storage.File{
-			Path:         path,
-			ContentType:  mime.TypeByExtension(filepath.Ext(filename)),
-			LastModified: stat.ModTime(),
-			Etag:         etag,
-			Encodings: map[string]io.ReadSeekCloser{
-				"identity": f,
-			},
+	buf := new(buffer.Buffer)
+	if _, err := f.WriteTo(buf); err != nil {
+		return nil, errors.WithMessage(err, "could not read file")
+	}
+
+	file := &storage.File{
+		Path:         path,
+		ContentType:  mime.TypeByExtension(filepath.Ext(filename)),
+		LastModified: stat.ModTime(),
+		Etag:         etag,
+		Encodings: map[string]*buffer.Buffer{
+			"identity": buf,
 		},
 	}
 
@@ -58,10 +60,12 @@ func (r *Reader) OpenFile(path string, filename string) (*File, error) {
 
 			return nil, errors.WithMessagef(err, "could not stat file %s", filename+suffix)
 		}
-		file.Encodings[enc], err = os.Open(filename + suffix)
+		bytes, err := os.ReadFile(filename + suffix)
 		if err != nil {
 			return nil, errors.WithMessagef(err, "could not read file %s", filename+suffix)
 		}
+		buf := buffer.NewBuffer(bytes)
+		file.Encodings[enc] = buf
 	}
 
 	return file, nil
@@ -76,23 +80,12 @@ func etag(f io.Reader) (string, error) {
 	return fmt.Sprintf(`W/"%x"`, hash.Sum(nil)), nil
 }
 
-func (f *File) Close() error {
-	var errs []error
-	for _, enc := range f.Encodings {
-		if err := enc.Close(); err != nil {
-			errs = append(errs, err)
-		}
-	}
-
-	return errors.Join(errs...)
-}
-
 func pathNameToFileName(pathname string) string {
 	if strings.HasSuffix(pathname, "/") {
 		pathname = pathname + "index.html"
 	}
 
-	return pathname
+	return strings.TrimPrefix(pathname, "/")
 }
 
 func fileNameToPathName(filename string) string {
diff --git a/internal/storage/files/reader.go b/internal/storage/files/reader.go
index 425436b..fff37da 100644
--- a/internal/storage/files/reader.go
+++ b/internal/storage/files/reader.go
@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"go.alanpearce.eu/website/internal/storage"
 	"go.alanpearce.eu/x/log"
 
 	"gitlab.com/tozd/go/errors"
@@ -13,14 +14,14 @@ import (
 type Reader struct {
 	root  string
 	log   *log.Logger
-	files map[string]*File
+	files map[string]*storage.File
 }
 
 func NewReader(path string, log *log.Logger) (*Reader, error) {
 	r := &Reader{
 		root:  path,
 		log:   log,
-		files: make(map[string]*File),
+		files: make(map[string]*storage.File),
 	}
 	if err := r.registerContentFiles(); err != nil {
 		return nil, errors.WithMessagef(err, "registering content files")
@@ -69,8 +70,8 @@ func (r *Reader) registerContentFiles() error {
 	return nil
 }
 
-func (r *Reader) GetFile(urlPath string) *File {
-	return r.files[urlPath]
+func (r *Reader) GetFile(urlPath string) (*storage.File, error) {
+	return r.files[urlPath], nil
 }
 
 func (r *Reader) CanonicalisePath(path string) (cPath string, differs bool) {
diff --git a/internal/storage/files/writer.go b/internal/storage/files/writer.go
index ce498e7..40cf364 100644
--- a/internal/storage/files/writer.go
+++ b/internal/storage/files/writer.go
@@ -2,14 +2,15 @@ package files
 
 import (
 	"compress/gzip"
-	"context"
 	"io"
 	"os"
 	"path/filepath"
+	"time"
 
+	"go.alanpearce.eu/website/internal/buffer"
+	"go.alanpearce.eu/website/internal/storage"
 	"go.alanpearce.eu/x/log"
 
-	"github.com/a-h/templ"
 	"github.com/andybalholm/brotli"
 	"gitlab.com/tozd/go/errors"
 )
@@ -29,66 +30,49 @@ type Options struct {
 	Compress bool
 }
 
-func NewWriter(outputDirectory string, logger *log.Logger, opts *Options) *Files {
+func NewWriter(outputDirectory string, logger *log.Logger, opts *Options) (*Files, error) {
 	return &Files{
 		outputDirectory: outputDirectory,
 		options:         opts,
 		log:             logger,
+	}, nil
+}
+
+func (f *Files) NewFile(path string) *storage.File {
+	return &storage.File{
+		Path:         path,
+		ContentType:  "",
+		LastModified: time.Time{},
+		Etag:         "",
+		Encodings: map[string]*buffer.Buffer{
+			"identity": nil,
+		},
 	}
 }
 
-func (f *Files) Open(filename string) (io.ReadCloser, error) {
+func (f *Files) OpenRead(filename string) (io.ReadCloser, error) {
 	return os.Open(f.join(filename))
 }
 
-func (f *Files) OutputToFile(output io.Reader, filename string) error {
-	if err := f.Mkdirp(filepath.Dir(filename)); err != nil {
-		return err
-	}
-	f.log.Debug("outputting file", "filename", filename)
-	file, err := f.OpenFileAndVariants(filename)
-	if err != nil {
-		return errors.WithMessage(err, "could not open output file")
-	}
-	defer file.Close()
-
-	if _, err := io.Copy(file, output); err != nil {
-		return errors.WithMessage(err, "could not write output file")
-	}
-
-	return nil
+func (f *Files) OpenWrite(filename string) (io.WriteCloser, error) {
+	return openFileWrite(f.join(filename))
 }
 
-func (f *Files) RenderToFile(component templ.Component, filename string) error {
-	if err := f.Mkdirp(filepath.Dir(filename)); err != nil {
-		return err
-	}
-	f.log.Debug("outputting file", "filename", filename)
-	file, err := f.OpenFileAndVariants(filename)
+func (f *Files) Write(pathname string, content *buffer.Buffer) error {
+	filename := pathNameToFileName(pathname)
+	err := f.Mkdirp(filepath.Dir(filename))
 	if err != nil {
-		return errors.WithMessage(err, "could not open output file")
-	}
-	defer file.Close()
-
-	if err := component.Render(context.TODO(), file); err != nil {
-		return errors.WithMessage(err, "could not write output file")
+		return errors.WithMessage(err, "could not create directory")
 	}
 
-	return nil
-}
-
-func (f *Files) WriterToFile(writer io.WriterTo, filename string) error {
-	if err := f.Mkdirp(filepath.Dir(filename)); err != nil {
-		return err
-	}
-	f.log.Debug("outputting file", "filename", filename)
-	file, err := f.OpenFileAndVariants(filename)
+	fd, err := openFileWrite(f.join(filename))
 	if err != nil {
-		return errors.WithMessage(err, "could not open output file")
+		return errors.WithMessagef(err, "could not open output file")
 	}
-	defer file.Close()
+	defer fd.Close()
 
-	if _, err := writer.WriteTo(file); err != nil {
+	_, err = fd.Write(content.Bytes())
+	if err != nil {
 		return errors.WithMessage(err, "could not write output file")
 	}
 
@@ -170,7 +154,7 @@ func (f *Files) OpenFileAndVariants(filename string) (io.WriteCloser, error) {
 
 func (f *Files) Mkdirp(dir string) error {
 	f.log.Debug("creating directory", "dir", dir)
-	err := os.MkdirAll(filepath.Join(f.outputDirectory, dir), 0755)
+	err := os.MkdirAll(f.join(dir), 0755)
 
 	return errors.WithMessage(err, "could not create directory")
 }
diff --git a/internal/storage/interface.go b/internal/storage/interface.go
index 6c8f3cd..c167a49 100644
--- a/internal/storage/interface.go
+++ b/internal/storage/interface.go
@@ -1,21 +1,16 @@
 package storage
 
 import (
-	"io"
-
-	"github.com/a-h/templ"
+	"go.alanpearce.eu/website/internal/buffer"
 )
 
+type Reader interface {
+	GetFile(path string) (*File, error)
+	CanonicalisePath(path string) (cPath string, differs bool)
+}
+
 type Writer interface {
 	Mkdirp(path string) error
 
-	Open(filename string) (io.ReadCloser, error)
-
-	OpenFileAndVariants(filename string) (io.WriteCloser, error)
-
-	OutputToFile(output io.Reader, filename string) error
-
-	RenderToFile(component templ.Component, filename string) error
-
-	WriterToFile(writer io.WriterTo, filename string) error
+	Write(pathname string, content *buffer.Buffer) error
 }
diff --git a/internal/website/mux.go b/internal/website/mux.go
index 7b1db36..784eb8c 100644
--- a/internal/website/mux.go
+++ b/internal/website/mux.go
@@ -66,7 +66,13 @@ func NewMux(
 
 			return nil
 		}
-		file := reader.GetFile(urlPath)
+		file, err := reader.GetFile(urlPath)
+		if err != nil {
+			return &ihttp.Error{
+				Message: "Error reading file",
+				Code:    http.StatusInternalServerError,
+			}
+		}
 		if file == nil {
 			return &ihttp.Error{
 				Message: "File not found",