diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | go.mod | 10 | ||||
| -rw-r--r-- | go.sum | 36 | ||||
| -rw-r--r-- | internal/builder/builder.go | 8 | ||||
| -rw-r--r-- | internal/server/server.go | 7 | ||||
| -rw-r--r-- | internal/storage/interface.go | 2 | ||||
| -rw-r--r-- | internal/storage/sqlite/file.go | 20 | ||||
| -rw-r--r-- | internal/storage/sqlite/reader.go | 99 | ||||
| -rw-r--r-- | internal/storage/sqlite/writer.go | 212 | ||||
| -rw-r--r-- | internal/website/mux.go | 5 |
10 files changed, 391 insertions, 9 deletions
diff --git a/.gitignore b/.gitignore index 130d516..46ed913 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ go.work *_templ.go *_templ.txt /.env +/site.db diff --git a/go.mod b/go.mod index 7981a74..f710e37 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/yuin/goldmark v1.7.4 gitlab.com/tozd/go/errors v0.8.1 go.alanpearce.eu/x v0.0.0-20241203124832-a29434dba11a + modernc.org/sqlite v1.34.5 ) require ( @@ -48,24 +49,30 @@ require ( github.com/cyphar/filepath-securejoin v0.2.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/libdns/libdns v0.2.2 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mholt/acmez/v2 v2.0.3 // indirect github.com/microcosm-cc/bluemonday v1.0.26 // indirect github.com/miekg/dns v1.1.62 // indirect github.com/mittwald/go-powerdns v0.6.6 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/onsi/gomega v1.34.2 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/redis/go-redis/v9 v9.7.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect github.com/snabb/diagio v1.0.4 // indirect @@ -84,5 +91,8 @@ require ( golang.org/x/tools v0.27.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect moul.io/zapfilter v1.7.0 // indirect ) diff --git a/go.sum b/go.sum index 7c12d1a..9668cb8 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,8 @@ github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80N github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -102,7 +104,11 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= @@ -127,6 +133,8 @@ github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s= github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= github.com/libdns/powerdns v0.1.3 h1:rRD/P0g/9Ru8cu4eGxiLp8GrMZTkj+BnNwIevTkUphM= github.com/libdns/powerdns v0.1.3/go.mod h1:xUy794+JpPeN9tM6PC1JITdetgRfRnPH1UFTrd2Eu2Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mholt/acmez/v2 v2.0.3 h1:CgDBlEwg3QBp6s45tPQmFIBrkRIkBT4rW4orMM6p4sw= github.com/mholt/acmez/v2 v2.0.3/go.mod h1:pQ1ysaDeGrIMvJ9dfJMk5kJNkn7L2sb3UhyrX6Q91cw= github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= @@ -136,6 +144,8 @@ github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXb github.com/mittwald/go-powerdns v0.6.6 h1:yQcuszhl98+jJgELjD5ecfxCQWoshhnArexpwrwQxLY= github.com/mittwald/go-powerdns v0.6.6/go.mod h1:adWJ860laOgm14afg+7V0nCa5NQT37oEYe2HRhoS/CA= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= github.com/osdevisnot/sorvor v0.4.4 h1:hcMWsWOKpUtDUE3F7dra1Jf12ftLHfgDcxlyPeVlz0Y= @@ -151,6 +161,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= @@ -327,5 +339,29 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= moul.io/zapfilter v1.7.0 h1:7aFrG4N72bDH9a2BtYUuUaDS981Dxu3qybWfeqaeBDU= moul.io/zapfilter v1.7.0/go.mod h1:M+N2s+qZiA+bzRoyKMVRxyuERijS2ovi2pnMyiOGMvc= diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 68f970f..266ce56 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -16,7 +16,7 @@ import ( "go.alanpearce.eu/website/internal/content" "go.alanpearce.eu/website/internal/sitemap" "go.alanpearce.eu/website/internal/storage" - "go.alanpearce.eu/website/internal/storage/files" + "go.alanpearce.eu/website/internal/storage/sqlite" "go.alanpearce.eu/website/templates" "go.alanpearce.eu/x/log" @@ -27,6 +27,7 @@ import ( type IOConfig struct { Source string `conf:"default:.,short:s,flag:src"` Destination string `conf:"default:public,short:d,flag:dest"` + DBPath string `conf:"default:site.db,flag:db"` Development bool `conf:"default:false,flag:dev"` } @@ -267,11 +268,12 @@ func BuildSite(ioConfig *IOConfig, cfg *config.Config, log *log.Logger) (*Result templates.Setup() loadCSS(ioConfig.Source) - storage, err := files.NewWriter(ioConfig.Destination, log, &files.Options{ + var storage storage.Writer + storage, err := sqlite.NewWriter(ioConfig.DBPath, log, &sqlite.Options{ Compress: !ioConfig.Development, }) if err != nil { - return nil, errors.WithMessage(err, "could not create storage writer") + return nil, errors.WithMessage(err, "could not create storage") } return build(storage, ioConfig, cfg, log) diff --git a/internal/server/server.go b/internal/server/server.go index e4ff63b..3c26a7f 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -16,7 +16,7 @@ import ( "go.alanpearce.eu/website/internal/builder" cfg "go.alanpearce.eu/website/internal/config" - "go.alanpearce.eu/website/internal/storage/files" + "go.alanpearce.eu/website/internal/storage/sqlite" "go.alanpearce.eu/website/internal/vcs" "go.alanpearce.eu/website/internal/website" "go.alanpearce.eu/x/log" @@ -33,6 +33,7 @@ var ( ) type Config struct { + DBPath string `conf:"default:site.db"` Root string `conf:"default:public"` Redirect bool `conf:"default:true"` ListenAddress string `conf:"default:localhost"` @@ -177,9 +178,9 @@ func New(runtimeConfig *Config, log *log.Logger) (*Server, error) { loggingMux := http.NewServeMux() log.Debug("registering content files", "root", runtimeConfig.Root) - reader, err := files.NewReader(runtimeConfig.Root, log.Named("files")) + reader, err := sqlite.NewReader(runtimeConfig.DBPath, log.Named("sqlite")) if err != nil { - return nil, errors.WithMessage(err, "could not create file reader") + return nil, errors.WithMessage(err, "could not create sqlite reader") } mux, err := website.NewMux(config, reader, log.Named("website")) diff --git a/internal/storage/interface.go b/internal/storage/interface.go index c167a49..282a33b 100644 --- a/internal/storage/interface.go +++ b/internal/storage/interface.go @@ -6,7 +6,7 @@ import ( type Reader interface { GetFile(path string) (*File, error) - CanonicalisePath(path string) (cPath string, differs bool) + CanonicalisePath(path string) (string, bool) } type Writer interface { diff --git a/internal/storage/sqlite/file.go b/internal/storage/sqlite/file.go new file mode 100644 index 0000000..328d538 --- /dev/null +++ b/internal/storage/sqlite/file.go @@ -0,0 +1,20 @@ +package sqlite + +import ( + "strings" +) + +func pathNameToFileName(pathname string) string { + if strings.HasSuffix(pathname, "/") { + pathname = pathname + "index.html" + } + + return pathname +} + +func fileNameToPathName(filename string) string { + pathname := strings.TrimSuffix(filename, "index.html") + pathname = strings.TrimPrefix(pathname, "/") + + return "/" + pathname +} diff --git a/internal/storage/sqlite/reader.go b/internal/storage/sqlite/reader.go new file mode 100644 index 0000000..fe5da7e --- /dev/null +++ b/internal/storage/sqlite/reader.go @@ -0,0 +1,99 @@ +package sqlite + +import ( + "database/sql" + "strings" + "time" + + "gitlab.com/tozd/go/errors" + "go.alanpearce.eu/website/internal/buffer" + "go.alanpearce.eu/website/internal/storage" + "go.alanpearce.eu/x/log" +) + +type Reader struct { + db *sql.DB + log *log.Logger + queries struct { + getFile *sql.Stmt + } +} + +func NewReader(dbPath string, log *log.Logger) (r *Reader, err error) { + db, err := openDB(dbPath) + if err != nil { + return nil, errors.WithMessage(err, "could not open SQLite database") + } + + r = &Reader{ + log: log, + db: db, + } + r.queries.getFile, err = r.db.Prepare(` + SELECT + file.content_type, + file.last_modified, + file.etag, + content.body + FROM url + INNER JOIN file + ON file.url_id = url.id + INNER JOIN content + ON content.file_id = file.id + WHERE + url.path = ? + AND + content.encoding = 'identity' + `) + if err != nil { + return nil, errors.WithMessage(err, "preparing select statement") + } + + return r, nil +} + +func (r *Reader) GetFile(filename string) (*storage.File, error) { + file := &storage.File{ + Encodings: make(map[string]*buffer.Buffer, 1), + } + content := []byte{} + var unixTime int64 + err := r.queries.getFile.QueryRow(filename).Scan( + &file.ContentType, + &unixTime, + &file.Etag, + &content, + ) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, errors.WithMessage(err, "querying database") + } + + file.LastModified = time.Unix(unixTime, 0) + file.Encodings["identity"] = buffer.NewBuffer(content) + + return file, nil +} + +// TODO write specialised query +func (r *Reader) CanonicalisePath(path string) (cPath string, differs bool) { + cPath = path + if strings.HasSuffix(path, "/index.html") { + cPath, differs = strings.CutSuffix(path, "index.html") + } else if !strings.HasSuffix(path, "/") { + file, err := r.GetFile(path + "/") + if err != nil { + r.log.Warn("error canonicalising path", "path", path, "error", err) + + return + } + + if file != nil { + cPath, differs = path+"/", true + } + } + + return +} diff --git a/internal/storage/sqlite/writer.go b/internal/storage/sqlite/writer.go new file mode 100644 index 0000000..c35494d --- /dev/null +++ b/internal/storage/sqlite/writer.go @@ -0,0 +1,212 @@ +package sqlite + +import ( + "database/sql" + "fmt" + "hash/fnv" + "mime" + "path/filepath" + "time" + + "go.alanpearce.eu/website/internal/buffer" + "go.alanpearce.eu/website/internal/storage" + "go.alanpearce.eu/x/log" + + "gitlab.com/tozd/go/errors" + _ "modernc.org/sqlite" +) + +type Writer struct { + db *sql.DB + + options *Options + log *log.Logger + queries struct { + insertURL *sql.Stmt + insertFile *sql.Stmt + insertContent *sql.Stmt + } +} + +type Options struct { + Compress bool +} + +func openDB(dbPath string) (*sql.DB, error) { + return sql.Open( + "sqlite", + fmt.Sprintf( + "file:%s?mode=%s&_pragma=foreign_keys(1)&_pragma=mmap_size(%d)", + dbPath, + "rwc", + 16*1024*1024, + ), + ) +} + +func NewWriter(dbPath string, logger *log.Logger, opts *Options) (*Writer, error) { + db, err := openDB(dbPath) + if err != nil { + return nil, errors.WithMessage(err, "opening sqlite database") + } + + // WIP: only memory database for now + _, err = db.Exec(` + CREATE TABLE IF NOT EXISTS url ( + id INTEGER PRIMARY KEY, + path TEXT NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS url_path + ON url (path); + + CREATE TABLE IF NOT EXISTS file ( + id INTEGER PRIMARY KEY, + url_id INTEGER NOT NULL, + content_type TEXT NOT NULL, + last_modified INTEGER NOT NULL, + etag TEXT NOT NULL, + FOREIGN KEY (url_id) REFERENCES url (id) + ); + CREATE UNIQUE INDEX IF NOT EXISTS file_url_content_type + ON file (url_id, content_type); + + CREATE TABLE IF NOT EXISTS content ( + file_id INTEGER NOT NULL, + encoding TEXT NOT NULL, + body BLOB NOT NULL, + FOREIGN KEY (file_id) REFERENCES file (id) + ); + CREATE UNIQUE INDEX IF NOT EXISTS file_content + ON content (file_id, encoding); + + CREATE TABLE IF NOT EXISTS redirect ( + id INTEGER PRIMARY KEY, + url_id INTEGER NOT NULL, + permanent BOOLEAN NOT NULL, + expires INTEGER, + FOREIGN KEY (url_id) REFERENCES url (id) + ); + `) + if err != nil { + return nil, errors.WithMessage(err, "creating tables") + } + + w := &Writer{ + db: db, + log: logger, + options: opts, + } + + w.queries.insertURL, err = db.Prepare(`INSERT INTO url (path) VALUES (?)`) + if err != nil { + return nil, errors.WithMessage(err, "preparing insert URL statement") + } + + w.queries.insertFile, err = db.Prepare(` + INSERT INTO file (url_id, content_type, last_modified, etag) + VALUES (:url_id, :content_type, :last_modified, :etag) + `) + if err != nil { + return nil, errors.WithMessage(err, "preparing insert file statement") + } + + w.queries.insertContent, err = db.Prepare(` + INSERT INTO content (file_id, encoding, body) + VALUES (:file_id, :encoding, :body) + `) + if err != nil { + return nil, errors.WithMessage(err, "preparing insert content statement") + } + + return w, nil +} + +func (s *Writer) Mkdirp(path string) error { + return nil +} + +func (s *Writer) storeURL(path string) (int64, error) { + r, err := s.queries.insertURL.Exec(path) + if err != nil { + return 0, errors.WithMessage(err, "inserting URL into database") + } + + return r.LastInsertId() +} + +func (s *Writer) storeFile(urlID int64, file *storage.File) (int64, error) { + r, err := s.queries.insertFile.Exec( + sql.Named("url_id", urlID), + sql.Named("content_type", file.ContentType), + sql.Named("last_modified", file.LastModified.Unix()), + sql.Named("etag", file.Etag), + ) + if err != nil { + return 0, errors.WithMessage(err, "inserting file into database") + } + + return r.LastInsertId() +} + +func (s *Writer) storeEncoding(fileID int64, encoding string, data []byte) error { + _, err := s.queries.insertContent.Exec( + sql.Named("file_id", fileID), + sql.Named("encoding", encoding), + sql.Named("body", data), + ) + if err != nil { + return errors.WithMessage(err, "inserting encoding into database") + } + + return nil +} + +func etag(content []byte) (string, error) { + hash := fnv.New64a() + hash.Write(content) + + return fmt.Sprintf(`W/"%x"`, hash.Sum(nil)), nil +} + +func contentType(pathname string) string { + return mime.TypeByExtension(filepath.Ext(pathNameToFileName(pathname))) +} + +func (s *Writer) Write(pathname string, content *buffer.Buffer) error { + s.log.Debug("storing content", "pathname", pathname) + bytes := content.Bytes() + + urlID, err := s.storeURL(pathname) + if err != nil { + return errors.WithMessage(err, "storing URL") + } + + etag, err := etag(bytes) + if err != nil { + return errors.WithMessage(err, "calculating etag") + } + + file := &storage.File{ + Path: pathname, + ContentType: contentType(pathname), + LastModified: time.Now(), + Etag: etag, + } + + fileID, err := s.storeFile(urlID, file) + if err != nil { + return errors.WithMessage(err, "storing file") + } + + err = s.storeEncoding(fileID, "identity", bytes) + if err != nil { + return errors.WithMessagef( + err, + "storing encoding file_id: %d, encoding: %s", + fileID, + "identity", + ) + } + + return nil +} diff --git a/internal/website/mux.go b/internal/website/mux.go index 784eb8c..05f6272 100644 --- a/internal/website/mux.go +++ b/internal/website/mux.go @@ -7,7 +7,7 @@ import ( "go.alanpearce.eu/website/internal/config" ihttp "go.alanpearce.eu/website/internal/http" - "go.alanpearce.eu/website/internal/storage/files" + "go.alanpearce.eu/website/internal/storage" "go.alanpearce.eu/website/templates" "go.alanpearce.eu/x/log" @@ -53,7 +53,7 @@ func (fn WrappedWebHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func NewMux( cfg *config.Config, - reader *files.Reader, + reader storage.Reader, log *log.Logger, ) (mux *http.ServeMux, err error) { mux = &http.ServeMux{} @@ -68,6 +68,7 @@ func NewMux( } file, err := reader.GetFile(urlPath) if err != nil { + log.Error("error getting file from reader", "err", err) return &ihttp.Error{ Message: "Error reading file", Code: http.StatusInternalServerError, |