move style hashing to file layer
16 files changed, 155 insertions(+), 134 deletions(-)
jump to
- cmd/build/main.go
- internal/builder/builder.go
- internal/builder/template/hasher.go
- internal/builder/template/template.go
- internal/server/server.go
- internal/storage/file.go
- internal/storage/files/file.go
- internal/storage/files/writer.go
- internal/storage/interface.go
- internal/storage/mime.go
- internal/storage/sqlite/reader.go
- internal/storage/sqlite/writer.go
- internal/website/mux.go
- internal/website/website.go
- templates/feed-styles.xsl
M cmd/build/main.go → cmd/build/main.go
@@ -94,12 +94,8 @@ if err != nil { log.Error("could not read config", "error", err) } - r, err := builder.BuildSite(options.Options, cfg, log) + err = builder.BuildSite(options.Options, cfg, log) if err != nil { panic("could not build site: " + err.Error()) - } - - for _, w := range r.Hashes { - fmt.Printf("\"'%s'\"\n", w) } }
M internal/builder/builder.go → internal/builder/builder.go
@@ -11,6 +11,7 @@ "slices" "time" "go.alanpearce.eu/homestead/internal/buffer" + "go.alanpearce.eu/homestead/internal/builder/template" "go.alanpearce.eu/homestead/internal/config" "go.alanpearce.eu/homestead/internal/content" "go.alanpearce.eu/homestead/internal/sitemap"@@ -30,10 +31,6 @@ VCSRemoteURL config.URL `conf:"default:https://git.alanpearce.eu/website"` Storage storage.Writer `conf:"-"` Repo *vcs.Repository `conf:"-"` -} - -type Result struct { - Hashes []string } func joinSourcePath(src string) func(string) string {@@ -65,16 +62,12 @@ func build( options *Options, config *config.Config, log *log.Logger, -) (*Result, error) { +) error { ctx := context.TODO() buf := new(buffer.Buffer) joinSource := joinSourcePath(options.Source) storage := options.Storage - r := &Result{ - Hashes: make([]string, 0), - } - log.Debug("reading posts", "source", options.Source) cc, err := content.NewContentCollection(&content.Config{ Root: options.Source,@@ -82,7 +75,7 @@ PostDir: "post", Repo: options.Repo, }, log.Named("content")) if err != nil { - return nil, err + return err } sitemap := sitemap.New(config)@@ -96,21 +89,21 @@ log.Debug("rendering post", "post", post.Basename) sitemap.AddPath(post.URL, post.Date) buf.Reset() if err := templates.PostPage(config, post).Render(ctx, buf); err != nil { - return nil, errors.WithMessage(err, "could not render post") + return errors.WithMessage(err, "could not render post") } if err := storage.WritePost(&post, buf); err != nil { - return nil, err + return err } } log.Debug("rendering tags list") buf.Reset() if err := templates.TagsPage(config, "tags", mapset.Sorted(cc.Tags), "/tags").Render(ctx, buf); err != nil { - return nil, err + return err } if err := storage.Write("/tags/", buf); err != nil { - return nil, err + return err } sitemap.AddPath("/tags/", lastMod)@@ -125,100 +118,93 @@ log.Debug("rendering tags page", "tag", tag) url := path.Join("/tags", tag) + "/" buf.Reset() if err := templates.TagPage(config, tag, matchingPosts, url).Render(ctx, buf); err != nil { - return nil, err + return err } if err = storage.Write(url, buf); err != nil { - return nil, err + return err } sitemap.AddPath(url, matchingPosts[0].Date) log.Debug("rendering tags feed", "tag", tag) - feed, err := renderFeed( + feed, err := template.RenderFeed( fmt.Sprintf("%s - %s", config.Title, tag), config, matchingPosts, tag, ) if err != nil { - return nil, errors.WithMessage(err, "could not render tag feed page") + return errors.WithMessage(err, "could not render tag feed page") } buf.Reset() if _, err := feed.WriteTo(buf); err != nil { - return nil, err + return err } if err := storage.Write(path.Join("/tags", tag, "atom.xml"), buf); err != nil { - return nil, err + return err } } log.Debug("rendering list page") buf.Reset() if err := templates.ListPage(config, cc.Posts, "/post").Render(ctx, buf); err != nil { - return nil, err + return err } if err := storage.Write("/post/", buf); err != nil { - return nil, err + return err } sitemap.AddPath("/post/", lastMod) log.Debug("rendering feed") - feed, err := renderFeed(config.Title, config, cc.Posts, "feed") + feed, err := template.RenderFeed(config.Title, config, cc.Posts, "feed") if err != nil { - return nil, errors.WithMessage(err, "could not render feed") + return errors.WithMessage(err, "could not render feed") } buf.Reset() if _, err := feed.WriteTo(buf); err != nil { - return nil, err + return err } if err := storage.Write("/atom.xml", buf); err != nil { - return nil, err + return err } - buf.Reset() - log.Debug("rendering feed styles") - if err := renderFeedStyles(buf); err != nil { - return nil, errors.WithMessage(err, "could not render feed styles") - } - if err := storage.Write("/feed-styles.xsl", buf); err != nil { - return nil, err - } - _, err = buf.Seek(0, 0) - if err != nil { - return nil, err + for _, filename := range []string{"feed-styles.xsl", "style.css"} { + buf.Reset() + log.Debug("rendering template file", "filename", filename) + if err := template.CopyFile(filename, buf); err != nil { + return errors.WithMessagef(err, "could not render template file %s", filename) + } + if err := storage.Write("/"+filename, buf); err != nil { + return err + } } - h, err := getFeedStylesHash(buf) - if err != nil { - return nil, err - } - r.Hashes = append(r.Hashes, h) pages, err := filepath.Glob(joinSource("*.md")) if err != nil { - return nil, errors.WithMessage(err, "could not glob pagess") + return errors.WithMessage(err, "could not glob pagess") } for _, p := range pages { page, err := filepath.Rel(options.Source, p) if err != nil { - return nil, err + return err } post, err := cc.GetPage(page) if err != nil { - return nil, err + return err } buf.Reset() log.Debug("rendering page", "source", page, "path", post.URL) if page == "index.md" { if err := templates.Homepage(config, cc.Posts, post).Render(ctx, buf); err != nil { - return nil, err + return err } } else { if err := templates.Page(config, post).Render(ctx, buf); err != nil { - return nil, err + return err } } if err := storage.WritePost(post, buf); err != nil { - return nil, err + return err } }@@ -228,31 +214,29 @@ // without a date, which could be newer sitemap.AddPath("/", time.Time{}) err = buf.SeekStart() if err != nil { - return nil, err + return err } - h, _ = getHTMLStyleHash(buf) - r.Hashes = append(r.Hashes, h) log.Debug("rendering sitemap") buf.Reset() if _, err := sitemap.WriteTo(buf); err != nil { - return nil, err + return err } if err := storage.Write("/sitemap.xml", buf); err != nil { - return nil, err + return err } log.Debug("rendering robots.txt") - rob, err := renderRobotsTXT(config) + rob, err := template.RenderRobotsTXT(config) if err != nil { - return nil, err + return err } buf.Reset() if _, err := io.Copy(buf, rob); err != nil { - return nil, err + return err } if err := storage.Write("/robots.txt", buf); err != nil { - return nil, err + return err } for _, sf := range cc.StaticFiles {@@ -260,16 +244,16 @@ src := joinSource(sf) log.Debug("copying static file", "sf", sf, "src", src) err = copyFile(storage, src, sf) if err != nil { - return nil, err + return err } } - return r, nil + return nil } -func BuildSite(options *Options, cfg *config.Config, log *log.Logger) (*Result, error) { +func BuildSite(options *Options, cfg *config.Config, log *log.Logger) error { if cfg == nil { - return nil, errors.New("config is nil") + return errors.New("config is nil") } cfg.InjectLiveReload = options.Development cfg.EnableGoatCounter = !options.Development
M internal/builder/hasher.go → internal/builder/template/hasher.go
@@ -1,11 +1,11 @@ -package builder +package template import ( "crypto/sha256" "encoding/base64" ) -func hash(s string) string { +func Hash(s string) string { shasum := sha256.New() shasum.Write([]byte(s))
M internal/builder/template.go → internal/builder/template/template.go
@@ -1,10 +1,9 @@ -package builder +package template import ( "bytes" "encoding/xml" "io" - "strings" "text/template" "go.alanpearce.eu/homestead/internal/atom"@@ -44,7 +43,7 @@ func (q *QueryDocument) Find(selector string) *QuerySelection { return &QuerySelection{q.Document.Find(selector)} } -func renderRobotsTXT(config *config.Config) (io.Reader, error) { +func RenderRobotsTXT(config *config.Config) (io.Reader, error) { r, w := io.Pipe() tpl, err := template.ParseFS(templates.Files, "robots.tmpl") if err != nil {@@ -63,7 +62,7 @@ return r, nil } -func renderFeed( +func RenderFeed( title string, config *config.Config, posts []content.Post,@@ -112,30 +111,21 @@ return buf, nil } -func renderFeedStyles(w io.Writer) error { - tpl, err := template.ParseFS(templates.Files, "feed-styles.xsl") - if err != nil { - return err - } - - esc := &strings.Builder{} - err = xml.EscapeText(esc, []byte(templates.CSS)) - +func CopyFile(filename string, w io.Writer) error { + f, err := templates.Files.Open(filename) if err != nil { return err } + defer f.Close() - err = tpl.Execute(w, map[string]interface{}{ - "css": esc.String(), - }) - if err != nil { + if _, err := io.Copy(w, f); err != nil { return err } return nil } -func getFeedStylesHash(r io.Reader) (string, error) { +func GetFeedStyleHash(r io.Reader) (string, error) { doc, err := xmlquery.Parse(r) if err != nil { return "", err@@ -146,15 +136,15 @@ return "", errors.WithMessage(err, "could not parse XPath") } style := xmlquery.QuerySelector(doc, expr) - return hash(style.InnerText()), nil + return Hash(style.InnerText()), nil } -func getHTMLStyleHash(r io.Reader) (string, error) { +func GetHTMLStyleHash(r io.Reader) (string, error) { doc, err := NewDocumentFromReader(r) if err != nil { return "", err } html := doc.Find("head > style").Text() - return hash(html), nil + return Hash(html), nil }
D internal/server/mime.go
@@ -1,19 +0,0 @@ -package server - -import ( - "mime" - - "go.alanpearce.eu/x/log" -) - -var newMIMEs = map[string]string{ - ".xsl": "text/xsl", -} - -func fixupMIMETypes(log *log.Logger) { - for ext, newType := range newMIMEs { - if err := mime.AddExtensionType(ext, newType); err != nil { - log.Error("could not update mime type", "ext", ext, "mime", newType) - } - } -}
M internal/server/server.go → internal/server/server.go
@@ -49,8 +49,6 @@ }) } func New(options *Options, log *log.Logger) (*Server, error) { - fixupMIMETypes(log) - return &Server{ mux: http.NewServeMux(), log: log,
M internal/storage/file.go → internal/storage/file.go
@@ -2,7 +2,12 @@ package storage import ( "io" + "strings" "time" + + "gitlab.com/tozd/go/errors" + "go.alanpearce.eu/homestead/internal/buffer" + "go.alanpearce.eu/homestead/internal/builder/template" ) type File struct {@@ -10,7 +15,8 @@ Path string ContentType string LastModified time.Time Etag string - Encodings map[string]io.ReadSeeker + StyleHash string + Encodings map[string]*buffer.Buffer } func (f *File) AvailableEncodings() []string {@@ -21,3 +27,31 @@ } return encs } + +func (f *File) CalculateStyleHash() (err error) { + buf := f.Encodings["identity"] + if buf == nil { + return errors.New("buffer not initialised") + } + _, err = buf.Seek(0, io.SeekStart) + if err != nil { + return errors.WithMessage(err, "could not seek buffer") + } + + mime, _, _ := strings.Cut(f.ContentType, ";") + switch mime { + case "text/html": + f.StyleHash, err = template.GetHTMLStyleHash(buf) + if err != nil { + return errors.WithMessage(err, "could not calculate HTML style hash") + } + default: + return + } + _, err = buf.Seek(0, io.SeekStart) + if err != nil { + return errors.WithMessage(err, "could not seek buffer") + } + + return +}
M internal/storage/files/file.go → internal/storage/files/file.go
@@ -46,9 +46,14 @@ Path: path, ContentType: mime.TypeByExtension(filepath.Ext(filename)), LastModified: stat.ModTime(), Etag: etag, - Encodings: map[string]io.ReadSeeker{ + Encodings: map[string]*buffer.Buffer{ "identity": buf, }, + } + + err = file.CalculateStyleHash() + if err != nil { + return nil, err } for enc, suffix := range encodings {
M internal/storage/files/writer.go → internal/storage/files/writer.go
@@ -9,6 +9,7 @@ "go.alanpearce.eu/homestead/internal/buffer" "go.alanpearce.eu/homestead/internal/content" "go.alanpearce.eu/homestead/internal/multifile" + "go.alanpearce.eu/homestead/internal/storage" "go.alanpearce.eu/x/log" "github.com/andybalholm/brotli"@@ -77,6 +78,10 @@ } fd.Close() return nil +} + +func (f *Files) WriteFile(file *storage.File, content *buffer.Buffer) error { + return f.Write(file.Path, content) } func (f *Files) write(pathname string, content *buffer.Buffer) (multifile.FileLike, error) {
M internal/storage/interface.go → internal/storage/interface.go
@@ -15,4 +15,5 @@ Mkdirp(path string) error Write(pathname string, content *buffer.Buffer) error WritePost(post *content.Post, content *buffer.Buffer) error + WriteFile(file *File, content *buffer.Buffer) error }
A internal/storage/mime.go
@@ -0,0 +1,19 @@ +package storage + +import ( + "log" + "mime" +) + +var newMIMEs = map[string]string{ + ".xsl": "text/xsl", + ".toml": "application/toml", +} + +func init() { + for ext, newType := range newMIMEs { + if err := mime.AddExtensionType(ext, newType); err != nil { + log.Panicf("could not update mime type %s for extension %s, mime", newType, ext) + } + } +}
M internal/storage/sqlite/reader.go → internal/storage/sqlite/reader.go
@@ -2,7 +2,6 @@ package sqlite import ( "database/sql" - "io" "strings" "time"@@ -31,6 +30,7 @@ SELECT file.content_type, file.last_modified, file.etag, + file.style_hash, content.encoding, content.body FROM url@@ -61,7 +61,7 @@ } func (r *Reader) GetFile(filename string) (*storage.File, error) { file := &storage.File{ - Encodings: make(map[string]io.ReadSeeker, 1), + Encodings: make(map[string]*buffer.Buffer, 1), } var unixTime int64 var encoding string@@ -80,6 +80,7 @@ err = rows.Scan( &file.ContentType, &unixTime, &file.Etag, + &file.StyleHash, &encoding, &content, )
M internal/storage/sqlite/writer.go → internal/storage/sqlite/writer.go
@@ -66,6 +66,7 @@ url_id INTEGER NOT NULL, content_type TEXT NOT NULL, last_modified INTEGER NOT NULL, etag TEXT NOT NULL, + style_hash TEXT NOT NULL, FOREIGN KEY (url_id) REFERENCES url (url_id) ); CREATE UNIQUE INDEX IF NOT EXISTS file_url_content_type@@ -97,8 +98,8 @@ return nil, errors.WithMessage(err, "preparing insert URL statement") } w.queries.insertFile, err = db.Prepare(` - INSERT INTO file (url_id, content_type, last_modified, etag) - VALUES (:url_id, :content_type, :last_modified, :etag) + INSERT INTO file (url_id, content_type, last_modified, etag, style_hash) + VALUES (:url_id, :content_type, :last_modified, :etag, :style_hash) `) if err != nil { return nil, errors.WithMessage(err, "preparing insert file statement")@@ -134,6 +135,7 @@ sql.Named("url_id", urlID), sql.Named("content_type", file.ContentType), sql.Named("last_modified", file.LastModified.Unix()), sql.Named("etag", file.Etag), + sql.Named("style_hash", file.StyleHash), ) if err != nil { return 0, errors.WithMessage(err, "inserting file into database")@@ -184,9 +186,10 @@ Path: post.URL, ContentType: contentType(post.URL), LastModified: post.Date, Etag: etag, + Encodings: map[string]*buffer.Buffer{}, } - return s.write(file, content) + return s.WriteFile(file, content) } func (s *Writer) Write(pathname string, content *buffer.Buffer) error {@@ -202,26 +205,36 @@ Path: pathname, ContentType: contentType(pathname), LastModified: time.Now(), Etag: etag, + Encodings: map[string]*buffer.Buffer{}, } - return s.write(file, content) + return s.WriteFile(file, content) } -func (s *Writer) write(file *storage.File, content *buffer.Buffer) error { +func (s *Writer) WriteFile(file *storage.File, content *buffer.Buffer) error { s.log.Debug("storing content", "pathname", file.Path) - bytes := content.Bytes() urlID, err := s.storeURL(file.Path) if err != nil { return errors.WithMessage(err, "storing URL") } + if file.Encodings == nil { + file.Encodings = map[string]*buffer.Buffer{} + } + file.Encodings["identity"] = content + + err = file.CalculateStyleHash() + if err != nil { + return errors.WithMessage(err, "calculating file hash") + } + fileID, err := s.storeFile(urlID, file) if err != nil { return errors.WithMessage(err, "storing file") } - err = s.storeEncoding(fileID, "identity", bytes) + err = s.storeEncoding(fileID, "identity", content.Bytes()) if err != nil { return err }
M internal/website/mux.go → internal/website/mux.go
@@ -52,7 +52,11 @@ } } w.Header().Add("ETag", file.Etag) w.Header().Add("Vary", "Accept-Encoding") - w.Header().Add("Content-Security-Policy", website.config.CSP.String()) + csp := *website.config.CSP + if file.StyleHash != "" { + csp.StyleSrc = []string{"'" + file.StyleHash + "'"} + } + w.Header().Add("Content-Security-Policy", csp.String()) for k, v := range website.config.Extra.Headers { w.Header().Add(k, v) }
M internal/website/website.go → internal/website/website.go
@@ -1,7 +1,6 @@ package website import ( - "fmt" "net/http" "slices" "strings"@@ -189,18 +188,11 @@ return website, nil } -func updateCSPHashes(config *config.Config, r *builder.Result) { - for i, h := range r.Hashes { - config.CSP.StyleSrc[i] = fmt.Sprintf("'%s'", h) - } -} - func rebuild(builderConfig *builder.Options, config *config.Config, log *log.Logger) error { - r, err := builder.BuildSite(builderConfig, config, log.Named("builder")) + err := builder.BuildSite(builderConfig, config, log.Named("builder")) if err != nil { return errors.WithMessage(err, "could not build site") } - updateCSPHashes(config, r) return nil }
M templates/feed-styles.xsl → templates/feed-styles.xsl
@@ -12,9 +12,7 @@ <title>RSS Feed | <xsl:value-of select="/atom:feed/atom:title"/></title> <meta charset="utf-8" /> <meta http-equiv="content-type" content="text/html; charset=utf-8" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> - <style> - {{ .css }} - </style> + <link rel="stylesheet" type="text/css" href="/style.css" /> </head> <body> <main>