switch to sqlite

1 files changed, 212 insertions, 0 deletions

diff --git a/internal/storage/sqlite/writer.go b/internal/storage/sqlite/writer.go
new file mode 100644
index 0000000..c35494d
--- /dev/null
+++ b/internal/storage/sqlite/writer.go
@@ -0,0 +1,212 @@
+package sqlite
+
+import (
+	"database/sql"
+	"fmt"
+	"hash/fnv"
+	"mime"
+	"path/filepath"
+	"time"
+
+	"go.alanpearce.eu/website/internal/buffer"
+	"go.alanpearce.eu/website/internal/storage"
+	"go.alanpearce.eu/x/log"
+
+	"gitlab.com/tozd/go/errors"
+	_ "modernc.org/sqlite"
+)
+
+type Writer struct {
+	db *sql.DB
+
+	options *Options
+	log     *log.Logger
+	queries struct {
+		insertURL     *sql.Stmt
+		insertFile    *sql.Stmt
+		insertContent *sql.Stmt
+	}
+}
+
+type Options struct {
+	Compress bool
+}
+
+func openDB(dbPath string) (*sql.DB, error) {
+	return sql.Open(
+		"sqlite",
+		fmt.Sprintf(
+			"file:%s?mode=%s&_pragma=foreign_keys(1)&_pragma=mmap_size(%d)",
+			dbPath,
+			"rwc",
+			16*1024*1024,
+		),
+	)
+}
+
+func NewWriter(dbPath string, logger *log.Logger, opts *Options) (*Writer, error) {
+	db, err := openDB(dbPath)
+	if err != nil {
+		return nil, errors.WithMessage(err, "opening sqlite database")
+	}
+
+	// WIP: only memory database for now
+	_, err = db.Exec(`
+		CREATE TABLE IF NOT EXISTS url (
+			id INTEGER PRIMARY KEY,
+			path TEXT NOT NULL
+		);
+		CREATE UNIQUE INDEX IF NOT EXISTS url_path
+			ON url (path);
+
+		CREATE TABLE IF NOT EXISTS file (
+			id INTEGER PRIMARY KEY,
+			url_id INTEGER NOT NULL,
+			content_type TEXT NOT NULL,
+			last_modified INTEGER NOT NULL,
+			etag TEXT NOT NULL,
+			FOREIGN KEY (url_id) REFERENCES url (id)
+		);
+		CREATE UNIQUE INDEX IF NOT EXISTS file_url_content_type
+			ON file (url_id, content_type);
+
+		CREATE TABLE IF NOT EXISTS content (
+			file_id INTEGER NOT NULL,
+			encoding TEXT NOT NULL,
+			body BLOB NOT NULL,
+			FOREIGN KEY (file_id) REFERENCES file (id)
+		);
+		CREATE UNIQUE INDEX IF NOT EXISTS file_content
+			ON content (file_id, encoding);
+
+		CREATE TABLE IF NOT EXISTS redirect (
+			id INTEGER PRIMARY KEY,
+			url_id INTEGER NOT NULL,
+			permanent BOOLEAN NOT NULL,
+			expires INTEGER,
+			FOREIGN KEY (url_id) REFERENCES url (id)
+		);
+	`)
+	if err != nil {
+		return nil, errors.WithMessage(err, "creating tables")
+	}
+
+	w := &Writer{
+		db:      db,
+		log:     logger,
+		options: opts,
+	}
+
+	w.queries.insertURL, err = db.Prepare(`INSERT INTO url (path) VALUES (?)`)
+	if err != nil {
+		return nil, errors.WithMessage(err, "preparing insert URL statement")
+	}
+
+	w.queries.insertFile, err = db.Prepare(`
+		INSERT INTO file (url_id, content_type, last_modified, etag)
+		VALUES (:url_id, :content_type, :last_modified, :etag)
+	`)
+	if err != nil {
+		return nil, errors.WithMessage(err, "preparing insert file statement")
+	}
+
+	w.queries.insertContent, err = db.Prepare(`
+		INSERT INTO content (file_id, encoding, body)
+		VALUES (:file_id, :encoding, :body)
+	`)
+	if err != nil {
+		return nil, errors.WithMessage(err, "preparing insert content statement")
+	}
+
+	return w, nil
+}
+
+func (s *Writer) Mkdirp(path string) error {
+	return nil
+}
+
+func (s *Writer) storeURL(path string) (int64, error) {
+	r, err := s.queries.insertURL.Exec(path)
+	if err != nil {
+		return 0, errors.WithMessage(err, "inserting URL into database")
+	}
+
+	return r.LastInsertId()
+}
+
+func (s *Writer) storeFile(urlID int64, file *storage.File) (int64, error) {
+	r, err := s.queries.insertFile.Exec(
+		sql.Named("url_id", urlID),
+		sql.Named("content_type", file.ContentType),
+		sql.Named("last_modified", file.LastModified.Unix()),
+		sql.Named("etag", file.Etag),
+	)
+	if err != nil {
+		return 0, errors.WithMessage(err, "inserting file into database")
+	}
+
+	return r.LastInsertId()
+}
+
+func (s *Writer) storeEncoding(fileID int64, encoding string, data []byte) error {
+	_, err := s.queries.insertContent.Exec(
+		sql.Named("file_id", fileID),
+		sql.Named("encoding", encoding),
+		sql.Named("body", data),
+	)
+	if err != nil {
+		return errors.WithMessage(err, "inserting encoding into database")
+	}
+
+	return nil
+}
+
+func etag(content []byte) (string, error) {
+	hash := fnv.New64a()
+	hash.Write(content)
+
+	return fmt.Sprintf(`W/"%x"`, hash.Sum(nil)), nil
+}
+
+func contentType(pathname string) string {
+	return mime.TypeByExtension(filepath.Ext(pathNameToFileName(pathname)))
+}
+
+func (s *Writer) Write(pathname string, content *buffer.Buffer) error {
+	s.log.Debug("storing content", "pathname", pathname)
+	bytes := content.Bytes()
+
+	urlID, err := s.storeURL(pathname)
+	if err != nil {
+		return errors.WithMessage(err, "storing URL")
+	}
+
+	etag, err := etag(bytes)
+	if err != nil {
+		return errors.WithMessage(err, "calculating etag")
+	}
+
+	file := &storage.File{
+		Path:         pathname,
+		ContentType:  contentType(pathname),
+		LastModified: time.Now(),
+		Etag:         etag,
+	}
+
+	fileID, err := s.storeFile(urlID, file)
+	if err != nil {
+		return errors.WithMessage(err, "storing file")
+	}
+
+	err = s.storeEncoding(fileID, "identity", bytes)
+	if err != nil {
+		return errors.WithMessagef(
+			err,
+			"storing encoding file_id: %d, encoding: %s",
+			fileID,
+			"identity",
+		)
+	}
+
+	return nil
+}