package sqlite import ( "database/sql" "fmt" "hash/fnv" "io" "mime" "path/filepath" "time" "github.com/andybalholm/brotli" "github.com/klauspost/compress/gzip" "github.com/klauspost/compress/zstd" "go.alanpearce.eu/website/internal/buffer" "go.alanpearce.eu/website/internal/content" "go.alanpearce.eu/website/internal/storage" "go.alanpearce.eu/x/log" "gitlab.com/tozd/go/errors" _ "modernc.org/sqlite" // import registers db/SQL driver ) var encodings = []string{"gzip", "br", "zstd"} type Writer struct { db *sql.DB options *Options log *log.Logger queries struct { insertURL *sql.Stmt insertFile *sql.Stmt insertContent *sql.Stmt } } type Options struct { Compress bool } func OpenDB(dbPath string) (*sql.DB, error) { return sql.Open( "sqlite", fmt.Sprintf( "file:%s?mode=%s&_pragma=foreign_keys(1)&_pragma=mmap_size(%d)", dbPath, "rwc", 16*1024*1024, ), ) } func NewWriter(db *sql.DB, logger *log.Logger, opts *Options) (*Writer, error) { _, err := db.Exec(` CREATE TABLE IF NOT EXISTS url ( url_id INTEGER PRIMARY KEY, path TEXT NOT NULL ); CREATE UNIQUE INDEX IF NOT EXISTS url_path ON url (path); CREATE TABLE IF NOT EXISTS file ( file_id INTEGER PRIMARY KEY, url_id INTEGER NOT NULL, content_type TEXT NOT NULL, last_modified INTEGER NOT NULL, etag TEXT NOT NULL, FOREIGN KEY (url_id) REFERENCES url (url_id) ); CREATE UNIQUE INDEX IF NOT EXISTS file_url_content_type ON file (url_id, content_type); CREATE TABLE IF NOT EXISTS content ( content_id INTEGER PRIMARY KEY, file_id INTEGER NOT NULL, encoding TEXT NOT NULL, body BLOB NOT NULL, FOREIGN KEY (file_id) REFERENCES file (file_id) ); CREATE UNIQUE INDEX IF NOT EXISTS file_content ON content (file_id, encoding); `) if err != nil { return nil, errors.WithMessage(err, "creating tables") } w := &Writer{ db: db, log: logger, options: opts, } w.queries.insertURL, err = db.Prepare(`INSERT INTO url (path) VALUES (?)`) if err != nil { return nil, errors.WithMessage(err, "preparing insert URL statement") } w.queries.insertFile, err = db.Prepare(` INSERT INTO file (url_id, content_type, last_modified, etag) VALUES (:url_id, :content_type, :last_modified, :etag) `) if err != nil { return nil, errors.WithMessage(err, "preparing insert file statement") } w.queries.insertContent, err = db.Prepare(` INSERT INTO content (file_id, encoding, body) VALUES (:file_id, :encoding, :body) `) if err != nil { return nil, errors.WithMessage(err, "preparing insert content statement") } return w, nil } func (s *Writer) Mkdirp(string) error { return nil } func (s *Writer) storeURL(path string) (int64, error) { r, err := s.queries.insertURL.Exec(path) if err != nil { return 0, errors.WithMessagef(err, "inserting URL %s into database", path) } return r.LastInsertId() } func (s *Writer) storeFile(urlID int64, file *storage.File) (int64, error) { r, err := s.queries.insertFile.Exec( sql.Named("url_id", urlID), sql.Named("content_type", file.ContentType), sql.Named("last_modified", file.LastModified.Unix()), sql.Named("etag", file.Etag), ) if err != nil { return 0, errors.WithMessage(err, "inserting file into database") } return r.LastInsertId() } func (s *Writer) storeEncoding(fileID int64, encoding string, data []byte) error { _, err := s.queries.insertContent.Exec( sql.Named("file_id", fileID), sql.Named("encoding", encoding), sql.Named("body", data), ) if err != nil { return errors.WithMessagef( err, "inserting encoding into database file_id: %d encoding: %s", fileID, encoding, ) } return nil } func etag(content []byte) (string, error) { hash := fnv.New64a() hash.Write(content) return fmt.Sprintf(`W/"%x"`, hash.Sum(nil)), nil } func contentType(pathname string) string { return mime.TypeByExtension(filepath.Ext(pathNameToFileName(pathname))) } func (s *Writer) WritePost(post *content.Post, content *buffer.Buffer) error { s.log.Debug("storing post", "title", post.Title) bytes := content.Bytes() etag, err := etag(bytes) if err != nil { return errors.WithMessage(err, "calculating etag") } file := &storage.File{ Path: post.URL, ContentType: contentType(post.URL), LastModified: post.Date, Etag: etag, } return s.write(file, content) } func (s *Writer) Write(pathname string, content *buffer.Buffer) error { bytes := content.Bytes() etag, err := etag(bytes) if err != nil { return errors.WithMessage(err, "calculating etag") } file := &storage.File{ Path: pathname, ContentType: contentType(pathname), LastModified: time.Now(), Etag: etag, } return s.write(file, content) } func (s *Writer) write(file *storage.File, content *buffer.Buffer) error { s.log.Debug("storing content", "pathname", file.Path) bytes := content.Bytes() urlID, err := s.storeURL(file.Path) if err != nil { return errors.WithMessage(err, "storing URL") } fileID, err := s.storeFile(urlID, file) if err != nil { return errors.WithMessage(err, "storing file") } err = s.storeEncoding(fileID, "identity", bytes) if err != nil { return err } if s.options.Compress { for _, enc := range encodings { compressed, err := compress(enc, content) if err != nil { return errors.WithMessage(err, "compressing file") } err = s.storeEncoding(fileID, enc, compressed.Bytes()) if err != nil { return err } } } return nil } func compress(encoding string, content *buffer.Buffer) (compressed *buffer.Buffer, err error) { var w io.WriteCloser compressed = new(buffer.Buffer) switch encoding { case "gzip": w = gzip.NewWriter(compressed) case "br": w = brotli.NewWriter(compressed) case "zstd": w, err = zstd.NewWriter(compressed) if err != nil { return nil, errors.WithMessage(err, "could not create zstd writer") } } defer w.Close() err = content.SeekStart() if err != nil { return nil, errors.WithMessage(err, "seeking to start of content buffer") } _, err = io.Copy(w, content) if err != nil { return nil, errors.WithMessage(err, "compressing file") } return compressed, nil }