package sqlite import ( "database/sql" "fmt" "hash/fnv" "mime" "path/filepath" "time" "go.alanpearce.eu/website/internal/buffer" "go.alanpearce.eu/website/internal/storage" "go.alanpearce.eu/x/log" "gitlab.com/tozd/go/errors" _ "modernc.org/sqlite" ) type Writer struct { db *sql.DB options *Options log *log.Logger queries struct { insertURL *sql.Stmt insertFile *sql.Stmt insertContent *sql.Stmt } } type Options struct { Compress bool } func openDB(dbPath string) (*sql.DB, error) { return sql.Open( "sqlite", fmt.Sprintf( "file:%s?mode=%s&_pragma=foreign_keys(1)&_pragma=mmap_size(%d)", dbPath, "rwc", 16*1024*1024, ), ) } func NewWriter(dbPath string, logger *log.Logger, opts *Options) (*Writer, error) { db, err := openDB(dbPath) if err != nil { return nil, errors.WithMessage(err, "opening sqlite database") } // WIP: only memory database for now _, err = db.Exec(` CREATE TABLE IF NOT EXISTS url ( id INTEGER PRIMARY KEY, path TEXT NOT NULL ); CREATE UNIQUE INDEX IF NOT EXISTS url_path ON url (path); CREATE TABLE IF NOT EXISTS file ( id INTEGER PRIMARY KEY, url_id INTEGER NOT NULL, content_type TEXT NOT NULL, last_modified INTEGER NOT NULL, etag TEXT NOT NULL, FOREIGN KEY (url_id) REFERENCES url (id) ); CREATE UNIQUE INDEX IF NOT EXISTS file_url_content_type ON file (url_id, content_type); CREATE TABLE IF NOT EXISTS content ( file_id INTEGER NOT NULL, encoding TEXT NOT NULL, body BLOB NOT NULL, FOREIGN KEY (file_id) REFERENCES file (id) ); CREATE UNIQUE INDEX IF NOT EXISTS file_content ON content (file_id, encoding); CREATE TABLE IF NOT EXISTS redirect ( id INTEGER PRIMARY KEY, url_id INTEGER NOT NULL, permanent BOOLEAN NOT NULL, expires INTEGER, FOREIGN KEY (url_id) REFERENCES url (id) ); `) if err != nil { return nil, errors.WithMessage(err, "creating tables") } w := &Writer{ db: db, log: logger, options: opts, } w.queries.insertURL, err = db.Prepare(`INSERT INTO url (path) VALUES (?)`) if err != nil { return nil, errors.WithMessage(err, "preparing insert URL statement") } w.queries.insertFile, err = db.Prepare(` INSERT INTO file (url_id, content_type, last_modified, etag) VALUES (:url_id, :content_type, :last_modified, :etag) `) if err != nil { return nil, errors.WithMessage(err, "preparing insert file statement") } w.queries.insertContent, err = db.Prepare(` INSERT INTO content (file_id, encoding, body) VALUES (:file_id, :encoding, :body) `) if err != nil { return nil, errors.WithMessage(err, "preparing insert content statement") } return w, nil } func (s *Writer) Mkdirp(path string) error { return nil } func (s *Writer) storeURL(path string) (int64, error) { r, err := s.queries.insertURL.Exec(path) if err != nil { return 0, errors.WithMessage(err, "inserting URL into database") } return r.LastInsertId() } func (s *Writer) storeFile(urlID int64, file *storage.File) (int64, error) { r, err := s.queries.insertFile.Exec( sql.Named("url_id", urlID), sql.Named("content_type", file.ContentType), sql.Named("last_modified", file.LastModified.Unix()), sql.Named("etag", file.Etag), ) if err != nil { return 0, errors.WithMessage(err, "inserting file into database") } return r.LastInsertId() } func (s *Writer) storeEncoding(fileID int64, encoding string, data []byte) error { _, err := s.queries.insertContent.Exec( sql.Named("file_id", fileID), sql.Named("encoding", encoding), sql.Named("body", data), ) if err != nil { return errors.WithMessage(err, "inserting encoding into database") } return nil } func etag(content []byte) (string, error) { hash := fnv.New64a() hash.Write(content) return fmt.Sprintf(`W/"%x"`, hash.Sum(nil)), nil } func contentType(pathname string) string { return mime.TypeByExtension(filepath.Ext(pathNameToFileName(pathname))) } func (s *Writer) Write(pathname string, content *buffer.Buffer) error { s.log.Debug("storing content", "pathname", pathname) bytes := content.Bytes() urlID, err := s.storeURL(pathname) if err != nil { return errors.WithMessage(err, "storing URL") } etag, err := etag(bytes) if err != nil { return errors.WithMessage(err, "calculating etag") } file := &storage.File{ Path: pathname, ContentType: contentType(pathname), LastModified: time.Now(), Etag: etag, } fileID, err := s.storeFile(urlID, file) if err != nil { return errors.WithMessage(err, "storing file") } err = s.storeEncoding(fileID, "identity", bytes) if err != nil { return errors.WithMessagef( err, "storing encoding file_id: %d, encoding: %s", fileID, "identity", ) } return nil }