switch to sqlite
11 files changed, 471 insertions(+), 19 deletions(-)
M go.mod → go.mod
@@ -25,6 +25,7 @@ github.com/stefanfritsch/goldmark-fences v1.0.0 github.com/yuin/goldmark v1.7.4 gitlab.com/tozd/go/errors v0.8.1 go.alanpearce.eu/x v0.0.0-20241203124832-a29434dba11a + modernc.org/sqlite v1.34.5 ) require (@@ -42,18 +43,24 @@ github.com/benpate/rosetta v0.21.2 // indirect github.com/cloudflare/circl v1.3.9 // indirect github.com/cyphar/filepath-securejoin v0.2.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/microcosm-cc/bluemonday v1.0.26 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/onsi/gomega v1.34.2 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect github.com/snabb/diagio v1.0.4 // indirect@@ -68,5 +75,8 @@ golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect moul.io/zapfilter v1.7.0 // indirect )
M go.sum → go.sum
@@ -59,6 +59,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=@@ -86,7 +88,11 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=@@ -105,8 +111,12 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= github.com/osdevisnot/sorvor v0.4.4 h1:hcMWsWOKpUtDUE3F7dra1Jf12ftLHfgDcxlyPeVlz0Y=@@ -120,6 +130,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=@@ -197,6 +209,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=@@ -223,6 +237,7 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=@@ -286,6 +301,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=@@ -307,5 +324,29 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= moul.io/zapfilter v1.7.0 h1:7aFrG4N72bDH9a2BtYUuUaDS981Dxu3qybWfeqaeBDU= moul.io/zapfilter v1.7.0/go.mod h1:M+N2s+qZiA+bzRoyKMVRxyuERijS2ovi2pnMyiOGMvc=
M internal/builder/builder.go → internal/builder/builder.go
@@ -2,6 +2,7 @@ package builder import ( "context" + "database/sql" "fmt" "io" "os"@@ -15,7 +16,7 @@ "go.alanpearce.eu/website/internal/config" "go.alanpearce.eu/website/internal/content" "go.alanpearce.eu/website/internal/sitemap" "go.alanpearce.eu/website/internal/storage" - "go.alanpearce.eu/website/internal/storage/files" + "go.alanpearce.eu/website/internal/storage/sqlite" "go.alanpearce.eu/website/internal/vcs" "go.alanpearce.eu/website/templates" "go.alanpearce.eu/x/log"@@ -26,9 +27,9 @@ ) type Options struct { Source string `conf:"default:.,short:s,flag:src"` - Destination string `conf:"default:public,short:d,flag:dest"` Development bool `conf:"default:false,flag:dev"` VCSRemoteURL config.URL `conf:"default:https://git.alanpearce.eu/website"` + DB *sql.DB `conf:"-"` Repo *vcs.Repository `conf:"-"` }@@ -278,8 +279,11 @@ } cfg.InjectLiveReload = options.Development cfg.EnableGoatCounter = !options.Development - storage, err := files.NewWriter(options.Destination, log, &files.Options{ - Compress: !options.Development, + templates.Init() + + var storage storage.Writer + storage, err := sqlite.NewWriter(options.DB, log.Named("storage"), &sqlite.Options{ + Compress: true, }) if err != nil { return nil, errors.WithMessage(err, "could not create storage writer")
M internal/server/server.go → internal/server/server.go
@@ -24,6 +24,8 @@ IdleTimeout = 10 * time.Minute ) type Options struct { + DBPath string `conf:"default:site.db"` + Redirect bool `conf:"default:true"` ListenAddress string `conf:"default:localhost"` Port int `conf:"default:8080,short:p"`
M internal/storage/interface.go → internal/storage/interface.go
@@ -6,7 +6,7 @@ ) type Reader interface { GetFile(path string) (*File, error) - CanonicalisePath(path string) (cPath string, differs bool) + CanonicalisePath(path string) (string, bool) } type Writer interface {
A internal/storage/sqlite/file.go
@@ -0,0 +1,13 @@ +package sqlite + +import ( + "strings" +) + +func pathNameToFileName(pathname string) string { + if strings.HasSuffix(pathname, "/") { + pathname += "index.html" + } + + return pathname +}
A internal/storage/sqlite/reader.go
@@ -0,0 +1,127 @@ +package sqlite + +import ( + "database/sql" + "io" + "strings" + "time" + + "gitlab.com/tozd/go/errors" + "go.alanpearce.eu/website/internal/buffer" + "go.alanpearce.eu/website/internal/storage" + "go.alanpearce.eu/x/log" +) + +type Reader struct { + db *sql.DB + log *log.Logger + queries struct { + getFile *sql.Stmt + checkPath *sql.Stmt + } +} + +func NewReader(db *sql.DB, log *log.Logger) (r *Reader, err error) { + r = &Reader{ + log: log, + db: db, + } + r.queries.getFile, err = r.db.Prepare(` + SELECT + file.content_type, + file.last_modified, + file.etag, + content.encoding, + content.body + FROM url + INNER JOIN file + USING (url_id) + INNER JOIN content + USING (file_id) + WHERE + url.path = ? + `) + if err != nil { + return nil, errors.WithMessage(err, "preparing select statement") + } + + r.queries.checkPath, err = r.db.Prepare(` + SELECT EXISTS( + SELECT 1 + FROM url + WHERE path = ? + ) AS differs +`) + if err != nil { + return nil, errors.WithMessage(err, "preparing check path statement") + } + + return r, nil +} + +func (r *Reader) GetFile(filename string) (*storage.File, error) { + file := &storage.File{ + Encodings: make(map[string]io.ReadSeeker, 1), + } + var unixTime int64 + var encoding string + var content []byte + rows, err := r.queries.getFile.Query(filename) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + + return nil, errors.WithMessage(err, "querying database") + } + + for rows.Next() { + err = rows.Scan( + &file.ContentType, + &unixTime, + &file.Etag, + &encoding, + &content, + ) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + + return nil, errors.WithMessage(err, "querying database") + } + + file.LastModified = time.Unix(unixTime, 0) + file.Encodings[encoding] = buffer.NewBuffer(content) + } + + return file, nil +} + +func (r *Reader) CanonicalisePath(path string) (cPath string, differs bool) { + cPath = path + switch { + case path == "/": + differs = false + case strings.HasSuffix(path, "/index.html"): + cPath, differs = strings.CutSuffix(path, "index.html") + case !strings.HasSuffix(path, "/"): + cPath += "/" + err := r.queries.checkPath.QueryRow(cPath).Scan(&differs) + if err != nil { + r.log.Warn("error canonicalising path", "path", path, "error", err) + + return + } + case strings.HasSuffix(path, "/"): + cPath = strings.TrimSuffix(path, "/") + err := r.queries.checkPath.QueryRow(cPath).Scan(&differs) + if err != nil { + r.log.Warn("error canonicalising path", "path", path, "error", err) + + return + } + } + + return +}
A internal/storage/sqlite/writer.go
@@ -0,0 +1,248 @@ +package sqlite + +import ( + "database/sql" + "fmt" + "hash/fnv" + "io" + "mime" + "path/filepath" + "time" + + "github.com/andybalholm/brotli" + "github.com/klauspost/compress/gzip" + "github.com/klauspost/compress/zstd" + "go.alanpearce.eu/website/internal/buffer" + "go.alanpearce.eu/website/internal/storage" + "go.alanpearce.eu/x/log" + + "gitlab.com/tozd/go/errors" + _ "modernc.org/sqlite" // import registers db/SQL driver +) + +var encodings = []string{"gzip", "br", "zstd"} + +type Writer struct { + db *sql.DB + + options *Options + log *log.Logger + queries struct { + insertURL *sql.Stmt + insertFile *sql.Stmt + insertContent *sql.Stmt + } +} + +type Options struct { + Compress bool +} + +func OpenDB(dbPath string) (*sql.DB, error) { + return sql.Open( + "sqlite", + fmt.Sprintf( + "file:%s?mode=%s&_pragma=foreign_keys(1)&_pragma=mmap_size(%d)", + dbPath, + "rwc", + 16*1024*1024, + ), + ) +} + +func NewWriter(db *sql.DB, logger *log.Logger, opts *Options) (*Writer, error) { + _, err := db.Exec(` + CREATE TABLE IF NOT EXISTS url ( + url_id INTEGER PRIMARY KEY, + path TEXT NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS url_path + ON url (path); + + CREATE TABLE IF NOT EXISTS file ( + file_id INTEGER PRIMARY KEY, + url_id INTEGER NOT NULL, + content_type TEXT NOT NULL, + last_modified INTEGER NOT NULL, + etag TEXT NOT NULL, + FOREIGN KEY (url_id) REFERENCES url (url_id) + ); + CREATE UNIQUE INDEX IF NOT EXISTS file_url_content_type + ON file (url_id, content_type); + + CREATE TABLE IF NOT EXISTS content ( + content_id INTEGER PRIMARY KEY, + file_id INTEGER NOT NULL, + encoding TEXT NOT NULL, + body BLOB NOT NULL, + FOREIGN KEY (file_id) REFERENCES file (file_id) + ); + CREATE UNIQUE INDEX IF NOT EXISTS file_content + ON content (file_id, encoding); + `) + if err != nil { + return nil, errors.WithMessage(err, "creating tables") + } + + w := &Writer{ + db: db, + log: logger, + options: opts, + } + + w.queries.insertURL, err = db.Prepare(`INSERT INTO url (path) VALUES (?)`) + if err != nil { + return nil, errors.WithMessage(err, "preparing insert URL statement") + } + + w.queries.insertFile, err = db.Prepare(` + INSERT INTO file (url_id, content_type, last_modified, etag) + VALUES (:url_id, :content_type, :last_modified, :etag) + `) + if err != nil { + return nil, errors.WithMessage(err, "preparing insert file statement") + } + + w.queries.insertContent, err = db.Prepare(` + INSERT INTO content (file_id, encoding, body) + VALUES (:file_id, :encoding, :body) + `) + if err != nil { + return nil, errors.WithMessage(err, "preparing insert content statement") + } + + return w, nil +} + +func (s *Writer) Mkdirp(string) error { + return nil +} + +func (s *Writer) storeURL(path string) (int64, error) { + r, err := s.queries.insertURL.Exec(path) + if err != nil { + return 0, errors.WithMessagef(err, "inserting URL %s into database", path) + } + + return r.LastInsertId() +} + +func (s *Writer) storeFile(urlID int64, file *storage.File) (int64, error) { + r, err := s.queries.insertFile.Exec( + sql.Named("url_id", urlID), + sql.Named("content_type", file.ContentType), + sql.Named("last_modified", file.LastModified.Unix()), + sql.Named("etag", file.Etag), + ) + if err != nil { + return 0, errors.WithMessage(err, "inserting file into database") + } + + return r.LastInsertId() +} + +func (s *Writer) storeEncoding(fileID int64, encoding string, data []byte) error { + _, err := s.queries.insertContent.Exec( + sql.Named("file_id", fileID), + sql.Named("encoding", encoding), + sql.Named("body", data), + ) + if err != nil { + return errors.WithMessagef( + err, + "inserting encoding into database file_id: %d encoding: %s", + fileID, + encoding, + ) + } + + return nil +} + +func etag(content []byte) (string, error) { + hash := fnv.New64a() + hash.Write(content) + + return fmt.Sprintf(`W/"%x"`, hash.Sum(nil)), nil +} + +func contentType(pathname string) string { + return mime.TypeByExtension(filepath.Ext(pathNameToFileName(pathname))) +} + +func (s *Writer) Write(pathname string, content *buffer.Buffer) error { + s.log.Debug("storing content", "pathname", pathname) + bytes := content.Bytes() + + urlID, err := s.storeURL(pathname) + if err != nil { + return errors.WithMessage(err, "storing URL") + } + + etag, err := etag(bytes) + if err != nil { + return errors.WithMessage(err, "calculating etag") + } + + file := &storage.File{ + Path: pathname, + ContentType: contentType(pathname), + LastModified: time.Now(), + Etag: etag, + } + + fileID, err := s.storeFile(urlID, file) + if err != nil { + return errors.WithMessage(err, "storing file") + } + + err = s.storeEncoding(fileID, "identity", bytes) + if err != nil { + return err + } + + if s.options.Compress { + for _, enc := range encodings { + compressed, err := compress(enc, content) + if err != nil { + return errors.WithMessage(err, "compressing file") + } + + err = s.storeEncoding(fileID, enc, compressed.Bytes()) + if err != nil { + return err + } + + } + } + + return nil +} + +func compress(encoding string, content *buffer.Buffer) (compressed *buffer.Buffer, err error) { + var w io.WriteCloser + compressed = new(buffer.Buffer) + switch encoding { + case "gzip": + w = gzip.NewWriter(compressed) + case "br": + w = brotli.NewWriter(compressed) + case "zstd": + w, err = zstd.NewWriter(compressed) + if err != nil { + return nil, errors.WithMessage(err, "could not create zstd writer") + } + } + defer w.Close() + + err = content.SeekStart() + if err != nil { + return nil, errors.WithMessage(err, "seeking to start of content buffer") + } + _, err = io.Copy(w, content) + if err != nil { + return nil, errors.WithMessage(err, "compressing file") + } + + return compressed, nil +}
M internal/website/mux.go → internal/website/mux.go
@@ -17,7 +17,7 @@ ) type Options struct { Source string `conf:"default:../website"` - Destination string `conf:"default:public"` + DBPath string `conf:"default:site.db"` Redirect bool `conf:"default:true"` Development bool `conf:"default:false,flag:dev"` BaseURL config.URL `conf:"default:localhost"`
M internal/website/website.go → internal/website/website.go
@@ -3,7 +3,6 @@ import ( "fmt" "net/http" - "os" "slices" "strings"@@ -13,7 +12,7 @@ "go.alanpearce.eu/website/internal/config" ihttp "go.alanpearce.eu/website/internal/http" "go.alanpearce.eu/website/internal/server" "go.alanpearce.eu/website/internal/storage" - "go.alanpearce.eu/website/internal/storage/files" + "go.alanpearce.eu/website/internal/storage/sqlite" "go.alanpearce.eu/website/internal/vcs" "go.alanpearce.eu/website/internal/watcher" "go.alanpearce.eu/website/templates"@@ -44,7 +43,6 @@ } builderOptions := &builder.Options{ Source: opts.Source, Development: opts.Development, - Destination: opts.Destination, } repo, exists, err := vcs.CloneOrOpen(&vcs.Options{@@ -82,16 +80,9 @@ } else { http.Error(w, err.Message, err.Code) } }) + if opts.Development { - tmpdir, err := os.MkdirTemp("", "website") - if err != nil { - log.Fatal("could not create temporary directory", "error", err) - } - log.Info("using temporary directory", "dir", tmpdir) - website.App.Shutdown = func() { - os.RemoveAll(tmpdir) - } - builderOptions.Destination = tmpdir + opts.DBPath = ":memory:" cfg.CSP.ScriptSrc = slices.Insert(cfg.CSP.ScriptSrc, 0, "'unsafe-inline'") cfg.CSP.ConnectSrc = slices.Insert(cfg.CSP.ConnectSrc, 0, "'self'")@@ -99,6 +90,11 @@ cfg.BaseURL = opts.BaseURL } + builderOptions.DB, err = sqlite.OpenDB(opts.DBPath) + if err != nil { + return nil, errors.WithMessage(err, "could not open database") + } + website.Domain = cfg.BaseURL.Hostname() err = rebuild(builderOptions, cfg, log)@@ -128,6 +124,16 @@ } go fw.Start(func(filename string) { log.Info("rebuilding site", "changed_file", filename) + builderOptions.DB.Close() + builderOptions.DB, err = sqlite.OpenDB(opts.DBPath) + if err != nil { + log.Error("error opening database", "error", err) + } + website.reader, err = sqlite.NewReader(builderOptions.DB, log.Named("reader")) + if err != nil { + log.Error("error creating sqlite reader", "error", err) + } + err := rebuild(builderOptions, cfg, log) if err != nil { log.Error("error rebuilding site", "error", err)@@ -136,7 +142,7 @@ opts.LiveReload.Reload() }) } - website.reader, err = files.NewReader(builderOptions.Destination, log.Named("reader")) + website.reader, err = sqlite.NewReader(builderOptions.DB, log.Named("reader")) if err != nil { return nil, errors.WithMessage(err, "error creating sqlite reader") }