package sqlite import ( "database/sql" "fmt" "hash/fnv" "io" "mime" "net/http" "path/filepath" "time" "github.com/andybalholm/brotli" "github.com/klauspost/compress/gzip" "github.com/klauspost/compress/zstd" "go.alanpearce.eu/homestead/internal/buffer" "go.alanpearce.eu/homestead/internal/content" "go.alanpearce.eu/homestead/internal/storage" "go.alanpearce.eu/x/log" "gitlab.com/tozd/go/errors" _ "modernc.org/sqlite" // import registers db/SQL driver ) var encodings = []string{"gzip", "br", "zstd"} type Writer struct { db *sql.DB options *Options log *log.Logger queries struct { insertURL *sql.Stmt insertFile *sql.Stmt insertContent *sql.Stmt } } type Options struct { Compress bool } func OpenDB(dbPath string) (*sql.DB, errors.E) { db, err := sql.Open( "sqlite", fmt.Sprintf( "file:%s?mode=%s&_pragma=foreign_keys(1)&_pragma=mmap_size(%d)", dbPath, "rwc", 16*1024*1024, ), ) if err != nil { return nil, errors.WithStack(err) } return db, nil } func NewWriter(db *sql.DB, logger *log.Logger, opts *Options) (*Writer, errors.E) { _, err := db.Exec(` CREATE TABLE IF NOT EXISTS url ( url_id INTEGER PRIMARY KEY, path TEXT NOT NULL ); CREATE UNIQUE INDEX IF NOT EXISTS url_path ON url (path); CREATE TABLE IF NOT EXISTS file ( file_id INTEGER PRIMARY KEY, url_id INTEGER NOT NULL, content_type TEXT NOT NULL, last_modified INTEGER NOT NULL, title TEXT NOT NULL, etag TEXT NOT NULL, style_hash TEXT NOT NULL, FOREIGN KEY (url_id) REFERENCES url (url_id) ); CREATE UNIQUE INDEX IF NOT EXISTS file_url_content_type ON file (url_id, content_type); CREATE TABLE IF NOT EXISTS content ( content_id INTEGER PRIMARY KEY, file_id INTEGER NOT NULL, encoding TEXT NOT NULL, body BLOB NOT NULL, FOREIGN KEY (file_id) REFERENCES file (file_id) ); CREATE UNIQUE INDEX IF NOT EXISTS file_content ON content (file_id, encoding); `) if err != nil { return nil, errors.WithMessage(err, "creating tables") } w := &Writer{ db: db, log: logger, options: opts, } w.queries.insertURL, err = db.Prepare(`INSERT INTO url (path) VALUES (?)`) if err != nil { return nil, errors.WithMessage(err, "preparing insert URL statement") } w.queries.insertFile, err = db.Prepare(` INSERT INTO file (url_id, content_type, last_modified, etag, style_hash, title) VALUES (:url_id, :content_type, :last_modified, :etag, :style_hash, :title) `) if err != nil { return nil, errors.WithMessage(err, "preparing insert file statement") } w.queries.insertContent, err = db.Prepare(` INSERT INTO content (file_id, encoding, body) VALUES (:file_id, :encoding, :body) `) if err != nil { return nil, errors.WithMessage(err, "preparing insert content statement") } return w, nil } func (s *Writer) Mkdirp(string) errors.E { return nil } func (s *Writer) storeURL(path string) (int64, errors.E) { r, err := s.queries.insertURL.Exec(path) if err != nil { return 0, errors.WithMessagef(err, "inserting URL %s into database", path) } id, err := r.LastInsertId() if err != nil { return 0, errors.WithStack(err) } return id, nil } func (s *Writer) storeFile(urlID int64, file *storage.File) (int64, errors.E) { if file.ContentType == "" { file.ContentType = http.DetectContentType(file.Encodings["identity"].Bytes()) s.log.Warn( "file has no content type, sniffing", "path", file.Path, "sniffed", file.ContentType, ) } r, err := s.queries.insertFile.Exec( sql.Named("url_id", urlID), sql.Named("content_type", file.ContentType), sql.Named("last_modified", file.LastModified.Unix()), sql.Named("etag", file.Etag), sql.Named("style_hash", file.StyleHash), sql.Named("title", file.Title), ) if err != nil { return 0, errors.WithMessage(err, "inserting file into database") } id, err := r.LastInsertId() if err != nil { return 0, errors.WithStack(err) } return id, nil } func (s *Writer) storeEncoding(fileID int64, encoding string, data []byte) errors.E { _, err := s.queries.insertContent.Exec( sql.Named("file_id", fileID), sql.Named("encoding", encoding), sql.Named("body", data), ) if err != nil { return errors.WithMessagef( err, "inserting encoding into database file_id: %d encoding: %s", fileID, encoding, ) } return nil } func etag(content []byte) (string, errors.E) { hash := fnv.New64a() _, err := hash.Write(content) if err != nil { return "", errors.WithStack(err) } return fmt.Sprintf(`W/"%x"`, hash.Sum(nil)), nil } func contentType(pathname string) string { return mime.TypeByExtension(filepath.Ext(pathNameToFileName(pathname))) } func (s *Writer) NewFileFromPost(post *content.Post) *storage.File { file := &storage.File{ Title: post.Title, Path: post.URL, LastModified: post.Date, Encodings: map[string]*buffer.Buffer{}, } return file } func (s *Writer) WritePost(post *content.Post, content *buffer.Buffer) errors.E { s.log.Debug("storing post", "title", post.Title) return s.WriteFile(s.NewFileFromPost(post), content) } func (s *Writer) Write(pathname string, title string, content *buffer.Buffer) errors.E { file := &storage.File{ Title: title, Path: pathname, LastModified: time.Now(), Encodings: map[string]*buffer.Buffer{}, } return s.WriteFile(file, content) } func (s *Writer) WriteFile(file *storage.File, content *buffer.Buffer) errors.E { s.log.Debug("storing content", "pathname", file.Path) urlID, err := s.storeURL(file.Path) if err != nil { return errors.WithMessage(err, "storing URL") } if file.Encodings == nil { file.Encodings = map[string]*buffer.Buffer{} } file.Encodings["identity"] = content if file.ContentType == "" { file.ContentType = contentType(file.Path) } if file.Etag == "" { file.Etag, err = etag(content.Bytes()) if err != nil { return errors.WithMessage(err, "could not calculate file etag") } } err = file.CalculateStyleHash() if err != nil { return errors.WithMessage(err, "calculating file hash") } fileID, err := s.storeFile(urlID, file) if err != nil { return errors.WithMessage(err, "storing file") } err = s.storeEncoding(fileID, "identity", content.Bytes()) if err != nil { return err } if s.options.Compress { for _, enc := range encodings { compressed, err := compress(enc, content) if err != nil { return errors.WithMessage(err, "compressing file") } err = s.storeEncoding(fileID, enc, compressed.Bytes()) if err != nil { return err } } } return nil } func compress(encoding string, content *buffer.Buffer) (*buffer.Buffer, errors.E) { var w io.WriteCloser compressed := new(buffer.Buffer) switch encoding { case "gzip": w = gzip.NewWriter(compressed) case "br": w = brotli.NewWriter(compressed) case "zstd": var err error w, err = zstd.NewWriter(compressed) if err != nil { return nil, errors.WithMessage(err, "could not create zstd writer") } } defer w.Close() if err := content.SeekStart(); err != nil { return nil, errors.WithMessage(err, "seeking to start of content buffer") } if _, err := io.Copy(w, content); err != nil { return nil, errors.WithMessage(err, "compressing file") } return compressed, nil }