From 2a4c795d5a165f995e9f7dc84e07465b140f3770 Mon Sep 17 00:00:00 2001 From: Alan Pearce Date: Sat, 27 Apr 2024 21:18:03 +0200 Subject: implement live-reloading dev server Squashed commit of the following: commit 02f077432202af4d633eb2cad81dfdaa6921317f Author: Alan Pearce Date: Sat Apr 27 21:09:14 2024 +0200 builder: only remove output directory if set and in dev mode commit 47001e01c55fa6e74aafeda04ebc3e4e7c47eba0 Author: Alan Pearce Date: Sat Apr 27 21:03:37 2024 +0200 implement live reload on dev server commit 411ec969f61e4b73439f1c54ea29f75135ecc618 Author: Alan Pearce Date: Sat Apr 27 20:59:26 2024 +0200 server: implement graceful shutdown commit 5400132eb6eb1b638e0b3fd4265f51611c92d473 Author: Alan Pearce Date: Sat Apr 27 20:41:07 2024 +0200 add some debug logs commit 3c9b678197c044603950232d222f501ef74d7873 Author: Alan Pearce Date: Sat Apr 27 20:39:09 2024 +0200 prefix log output with executable name commit 300e24c179e390e9d3f5aeab4471c97f17f1fa64 Author: Alan Pearce Date: Sat Apr 27 20:29:42 2024 +0200 don't panic inside internal packages, return error instead commit fe2715d330402ad67fe866471bed89c7238ad2ec Author: Alan Pearce Date: Fri Apr 26 01:18:29 2024 +0200 config: use a table to configure CSP headers commit d012553aaf78a436fa8871830b5d720a9e292d4b Author: Alan Pearce Date: Thu Apr 25 17:13:39 2024 +0200 dev: create basic dev server to build and serve from a temporary directory commit a1d11d3e69650d9b43ca1b1d7b7ccc05a808d5c1 Author: Alan Pearce Date: Thu Apr 25 13:02:22 2024 +0200 remove unused redirect_other_hostnames config option commit fd67b19b5c7f76f0c3579e8a05ef20a618e90be7 Author: Alan Pearce Date: Thu Apr 25 12:58:53 2024 +0200 server: make port a string, which is what go uses commit c798e8e736c0649008cade337158399470a9099b Author: Alan Pearce Date: Thu Apr 25 12:58:33 2024 +0200 config: remove unused port variable commit f94882b9001f3b0855e26b26b4a84b96e3deb22b Author: Alan Pearce Date: Thu Apr 25 12:49:10 2024 +0200 re-organise module layout --- internal/builder/builder.go | 197 +++++++++++++++++++++ internal/builder/posts.go | 121 +++++++++++++ internal/builder/template.go | 371 ++++++++++++++++++++++++++++++++++++++++ internal/config/config.go | 29 ++-- internal/config/csp.go | 44 +++++ internal/config/cspgenerator.go | 79 +++++++++ internal/server/filemap.go | 77 +++++++++ internal/server/logging.go | 55 ++++++ internal/server/server.go | 221 ++++++++++++++++++++++++ 9 files changed, 1176 insertions(+), 18 deletions(-) create mode 100644 internal/builder/builder.go create mode 100644 internal/builder/posts.go create mode 100644 internal/builder/template.go create mode 100644 internal/config/csp.go create mode 100644 internal/config/cspgenerator.go create mode 100644 internal/server/filemap.go create mode 100644 internal/server/logging.go create mode 100644 internal/server/server.go (limited to 'internal') diff --git a/internal/builder/builder.go b/internal/builder/builder.go new file mode 100644 index 0000000..88e3f02 --- /dev/null +++ b/internal/builder/builder.go @@ -0,0 +1,197 @@ +package builder + +import ( + "fmt" + "io" + "log" + "log/slog" + "net/url" + "os" + "path" + "slices" + + "website/internal/config" + + cp "github.com/otiai10/copy" + "github.com/pkg/errors" +) + +type IOConfig struct { + Source string `conf:"default:.,short:s,flag:src"` + Destination string `conf:"default:website,short:d,flag:dest"` + BaseURL config.URL + Development bool `conf:"default:false,flag:dev"` +} + +func mkdirp(dirs ...string) error { + return os.MkdirAll(path.Join(dirs...), 0755) +} + +func outputToFile(output io.Reader, filename ...string) error { + slog.Debug(fmt.Sprintf("outputting file %s", path.Join(filename...))) + file, err := os.OpenFile(path.Join(filename...), os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return errors.WithMessage(err, "could not open output file") + } + defer file.Close() + + if _, err := file.ReadFrom(output); err != nil { + return errors.WithMessage(err, "could not write output file") + } + return nil +} + +func build(outDir string, config config.Config) error { + slog.Debug(fmt.Sprintf("output directory %s", outDir)) + privateDir := path.Join(outDir, "private") + if err := mkdirp(privateDir); err != nil { + return errors.WithMessage(err, "could not create private directory") + } + publicDir := path.Join(outDir, "public") + if err := mkdirp(publicDir); err != nil { + return errors.WithMessage(err, "could not create public directory") + } + + err := cp.Copy("static", publicDir, cp.Options{ + PreserveTimes: true, + PermissionControl: cp.AddPermission(0755), + }) + if err != nil { + return errors.WithMessage(err, "could not copy static files") + } + + if err := mkdirp(publicDir, "post"); err != nil { + return errors.WithMessage(err, "could not create post output directory") + } + slog.Debug("reading posts") + posts, tags, err := readPosts("content", "post", publicDir) + if err != nil { + return err + } + + for _, post := range posts { + if err := mkdirp(publicDir, "post", post.Basename); err != nil { + return errors.WithMessage(err, "could not create directory for post") + } + slog.Debug("rendering post", "post", post.Basename) + output, err := renderPost(post, config) + if err != nil { + return errors.WithMessagef(err, "could not render post %s", post.Input) + } + if err := outputToFile(output, post.Output); err != nil { + return err + } + } + + if err := mkdirp(publicDir, "tags"); err != nil { + return errors.WithMessage(err, "could not create directory for tags") + } + slog.Debug("rendering tags list") + output, err := renderTags(tags, config, "/tags") + if err != nil { + return errors.WithMessage(err, "could not render tags") + } + if err := outputToFile(output, publicDir, "tags", "index.html"); err != nil { + return err + } + + for _, tag := range tags.ToSlice() { + matchingPosts := []Post{} + for _, post := range posts { + if slices.Contains(post.Taxonomies.Tags, tag) { + matchingPosts = append(matchingPosts, post) + } + } + if err := mkdirp(publicDir, "tags", tag); err != nil { + return errors.WithMessage(err, "could not create directory") + } + slog.Debug("rendering tags page", "tag", tag) + output, err := renderListPage(tag, config, matchingPosts, "/tags/"+tag) + if err != nil { + return errors.WithMessage(err, "could not render tag page") + } + if err := outputToFile(output, publicDir, "tags", tag, "index.html"); err != nil { + return err + } + + slog.Debug("rendering tags feed", "tag", tag) + output, err = renderFeed(fmt.Sprintf("%s - %s", config.Title, tag), config, matchingPosts, tag) + if err != nil { + return errors.WithMessage(err, "could not render tag feed page") + } + if err := outputToFile(output, publicDir, "tags", tag, "atom.xml"); err != nil { + return err + } + } + + slog.Debug("rendering list page") + listPage, err := renderListPage("", config, posts, "/post") + if err != nil { + return errors.WithMessage(err, "could not render list page") + } + if err := outputToFile(listPage, publicDir, "post", "index.html"); err != nil { + return err + } + + slog.Debug("rendering feed") + feed, err := renderFeed(config.Title, config, posts, "feed") + if err != nil { + return errors.WithMessage(err, "could not render feed") + } + if err := outputToFile(feed, publicDir, "atom.xml"); err != nil { + return err + } + + slog.Debug("rendering feed styles") + feedStyles, err := renderFeedStyles() + if err != nil { + return errors.WithMessage(err, "could not render feed styles") + } + if err := outputToFile(feedStyles, publicDir, "feed-styles.xsl"); err != nil { + return err + } + + slog.Debug("rendering homepage") + homePage, err := renderHomepage(config, posts, "/") + if err != nil { + return errors.WithMessage(err, "could not render homepage") + } + if err := outputToFile(homePage, publicDir, "index.html"); err != nil { + return err + } + + slog.Debug("rendering 404 page") + notFound, err := render404(config, "/404.html") + if err != nil { + return errors.WithMessage(err, "could not render 404 page") + } + if err := outputToFile(notFound, privateDir, "404.html"); err != nil { + return err + } + + return nil +} + +func BuildSite(ioConfig IOConfig) error { + config, err := config.GetConfig() + if err != nil { + log.Panic(errors.Errorf("could not get config: %v", err)) + } + config.InjectLiveReload = ioConfig.Development + + if ioConfig.BaseURL.URL != nil { + config.BaseURL.URL, err = url.Parse(ioConfig.BaseURL.String()) + if err != nil { + log.Panic(errors.Errorf("highly unlikely: %v", err)) + } + } + + if ioConfig.Development && ioConfig.Destination != "website" { + err = os.RemoveAll(ioConfig.Destination) + if err != nil { + log.Panic(errors.Errorf("could not remove destination directory: %v", err)) + } + } + + return build(ioConfig.Destination, *config) +} diff --git a/internal/builder/posts.go b/internal/builder/posts.go new file mode 100644 index 0000000..223531b --- /dev/null +++ b/internal/builder/posts.go @@ -0,0 +1,121 @@ +package builder + +import ( + "bytes" + "log/slog" + "os" + "path" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/adrg/frontmatter" + mapset "github.com/deckarep/golang-set/v2" + "github.com/pkg/errors" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/extension" + htmlrenderer "github.com/yuin/goldmark/renderer/html" +) + +type PostMatter struct { + Date time.Time `toml:"date"` + Description string `toml:"description"` + Title string `toml:"title"` + Taxonomies struct { + Tags []string `toml:"tags"` + } `toml:"taxonomies"` +} + +type Post struct { + Input string + Output string + Basename string + URL string + Content string + PostMatter +} + +type Tags mapset.Set[string] + +var markdown = goldmark.New( + goldmark.WithRendererOptions( + htmlrenderer.WithUnsafe(), + ), + goldmark.WithExtensions( + extension.GFM, + extension.Footnote, + extension.Typographer, + ), +) + +func getPost(filename string) (*PostMatter, []byte, error) { + matter := PostMatter{} + content, err := os.Open(filename) + defer content.Close() + if err != nil { + return nil, nil, errors.WithMessagef(err, "could not open post %s", filename) + } + rest, err := frontmatter.MustParse(content, &matter) + if err != nil { + return nil, nil, errors.WithMessagef(err, "could not parse front matter of post %s", filename) + } + + return &matter, rest, nil +} + +func renderMarkdown(content []byte) (string, error) { + var buf bytes.Buffer + if err := markdown.Convert(content, &buf); err != nil { + return "", errors.WithMessage(err, "could not convert markdown content") + } + return buf.String(), nil +} + +func readPosts(root string, inputDir string, outputDir string) ([]Post, Tags, error) { + tags := mapset.NewSet[string]() + posts := []Post{} + subdir := filepath.Join(root, inputDir) + files, err := os.ReadDir(subdir) + if err != nil { + return nil, nil, errors.WithMessagef(err, "could not read post directory %s", subdir) + } + outputReplacer := strings.NewReplacer(root, outputDir, ".md", "/index.html") + urlReplacer := strings.NewReplacer(root, "", ".md", "/") + for _, f := range files { + pathFromRoot := filepath.Join(subdir, f.Name()) + if !f.IsDir() && path.Ext(pathFromRoot) == ".md" { + output := outputReplacer.Replace(pathFromRoot) + url := urlReplacer.Replace(pathFromRoot) + slog.Debug("reading post", "post", pathFromRoot) + matter, content, err := getPost(pathFromRoot) + if err != nil { + return nil, nil, err + } + + for _, tag := range matter.Taxonomies.Tags { + tags.Add(strings.ToLower(tag)) + } + + slog.Debug("rendering markdown in post", "post", pathFromRoot) + html, err := renderMarkdown(content) + if err != nil { + return nil, nil, err + } + post := Post{ + Input: pathFromRoot, + Output: output, + Basename: filepath.Base(url), + URL: url, + PostMatter: *matter, + Content: html, + } + + posts = append(posts, post) + } + } + slices.SortFunc(posts, func(a, b Post) int { + return b.Date.Compare(a.Date) + }) + return posts, tags, nil +} diff --git a/internal/builder/template.go b/internal/builder/template.go new file mode 100644 index 0000000..74d0418 --- /dev/null +++ b/internal/builder/template.go @@ -0,0 +1,371 @@ +package builder + +import ( + "encoding/xml" + "fmt" + "io" + "log/slog" + "net/url" + "os" + "strings" + "sync" + "time" + "website/internal/atom" + "website/internal/config" + + "github.com/PuerkitoBio/goquery" + "github.com/a-h/htmlformat" + "github.com/antchfx/xmlquery" + "github.com/antchfx/xpath" + mapset "github.com/deckarep/golang-set/v2" + "golang.org/x/net/html" +) + +var ( + assetsOnce sync.Once + css string + countHTML *goquery.Document + liveReloadHTML *goquery.Document + templates map[string]*os.File = make(map[string]*os.File) +) + +func loadTemplate(path string) (file *os.File, err error) { + if templates[path] == nil { + file, err = os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + return nil, err + } + templates[path] = file + } + file = templates[path] + return +} + +var ( + imgOnce sync.Once + img *goquery.Selection + urlTemplate *url.URL +) + +type QuerySelection struct { + *goquery.Selection +} + +type QueryDocument struct { + *goquery.Document +} + +func NewDocumentFromReader(r io.Reader) (*QueryDocument, error) { + doc, err := goquery.NewDocumentFromReader(r) + return &QueryDocument{doc}, err +} + +func (q *QueryDocument) Find(selector string) *QuerySelection { + return &QuerySelection{q.Document.Find(selector)} +} + +func NewDocumentNoScript(r io.Reader) (*goquery.Document, error) { + root, err := html.ParseWithOptions(r, html.ParseOptionEnableScripting(false)) + return goquery.NewDocumentFromNode(root), err +} + +func (root QuerySelection) setImgURL(pageURL string, pageTitle string) QuerySelection { + clone := countHTML.Clone() + imgOnce.Do(func() { + var err error + img = clone.Find("img") + attr, _ := img.Attr("src") + if attr == "" { + panic(" does not have src attribute") + } + urlTemplate, err = url.Parse(attr) + if err != nil { + panic(err.Error()) + } + }) + q := urlTemplate.Query() + urlTemplate.RawQuery = "" + q.Set("p", pageURL) + q.Set("t", pageTitle) + output := urlTemplate.String() + "?" + q.Encode() + clone.Find("img").SetAttr("src", output) + root.AppendSelection(clone.Find("body").Children()) + return root +} + +func layout(filename string, config config.Config, pageTitle string, pageURL string) (*goquery.Document, error) { + html, err := loadTemplate(filename) + if err != nil { + return nil, err + } + defer html.Seek(0, io.SeekStart) + assetsOnce.Do(func() { + var bytes []byte + bytes, err = os.ReadFile("templates/style.css") + if err != nil { + return + } + css = string(bytes) + countFile, err := os.OpenFile("templates/count.html", os.O_RDONLY, 0) + if err != nil { + return + } + defer countFile.Close() + countHTML, err = NewDocumentNoScript(countFile) + if config.InjectLiveReload { + liveReloadFile, err := os.OpenFile("templates/dev.html", os.O_RDONLY, 0) + if err != nil { + return + } + defer liveReloadFile.Close() + liveReloadHTML, err = goquery.NewDocumentFromReader(liveReloadFile) + } + }) + if err != nil { + return nil, err + } + + doc, err := NewDocumentFromReader(html) + if err != nil { + return nil, err + } + doc.Find("html").SetAttr("lang", config.DefaultLanguage) + doc.Find("head > link[rel=alternate]").SetAttr("title", config.Title) + doc.Find("head > link[rel=canonical]").SetAttr("href", pageURL) + doc.Find(".title").SetText(config.Title) + doc.Find("title").Add(".p-name").SetText(pageTitle) + doc.Find("head > style").SetHtml(css) + doc.Find("body").setImgURL(pageURL, pageTitle) + if config.InjectLiveReload { + doc.Find("body").AppendSelection(liveReloadHTML.Find("body").Clone()) + } + nav := doc.Find("nav") + navLink := doc.Find("nav a") + nav.Empty() + for _, link := range config.Menus["main"] { + nav.AppendSelection(navLink.Clone().SetAttr("href", link.URL).SetText(link.Name)) + } + return doc.Document, nil +} + +func renderPost(post Post, config config.Config) (r io.Reader, err error) { + doc, err := layout("templates/post.html", config, post.PostMatter.Title, post.URL) + if err != nil { + return nil, err + } + doc.Find(".title").AddClass("p-author h-card").SetAttr("rel", "author") + doc.Find(".h-entry .dt-published"). + SetAttr("datetime", post.PostMatter.Date.UTC().Format(time.RFC3339)). + SetText( + post.PostMatter.Date.Format("2006-01-02"), + ) + doc.Find(".h-entry .e-content").SetHtml(post.Content) + categories := doc.Find(".h-entry .p-categories") + tpl := categories.Find(".p-category").ParentsUntilSelection(categories) + tpl.Remove() + for _, tag := range post.Taxonomies.Tags { + cat := tpl.Clone() + cat.Find(".p-category").SetAttr("href", fmt.Sprintf("/tags/%s/", tag)).SetText("#" + tag) + categories.AppendSelection(cat) + } + + return renderHTML(doc), nil +} + +func renderTags(tags Tags, config config.Config, url string) (io.Reader, error) { + doc, err := layout("templates/tags.html", config, config.Title, url) + if err != nil { + return nil, err + } + tagList := doc.Find(".tags") + tpl := doc.Find(".h-feed") + tpl.Remove() + for _, tag := range mapset.Sorted(tags) { + li := tpl.Clone() + li.Find("a").SetAttr("href", fmt.Sprintf("/tags/%s/", tag)).SetText("#" + tag) + tagList.AppendSelection(li) + } + return renderHTML(doc), nil +} + +func renderListPage(tag string, config config.Config, posts []Post, url string) (io.Reader, error) { + var title string + if len(tag) > 0 { + title = tag + } else { + title = config.Title + } + doc, err := layout("templates/list.html", config, title, url) + if err != nil { + return nil, err + } + feed := doc.Find(".h-feed") + tpl := feed.Find(".h-entry") + tpl.Remove() + + doc.Find(".title").AddClass("p-author h-card").SetAttr("rel", "author") + if tag == "" { + doc.Find(".filter").Remove() + } else { + doc.Find(".filter").Find("h3").SetText("#" + tag) + } + + for _, post := range posts { + entry := tpl.Clone() + entry.Find(".p-name").SetText(post.Title).SetAttr("href", post.URL) + entry.Find(".dt-published"). + SetAttr("datetime", post.PostMatter.Date.UTC().Format(time.RFC3339)). + SetText(post.PostMatter.Date.Format("2006-01-02")) + feed.AppendSelection(entry) + } + + return renderHTML(doc), nil +} + +func renderHomepage(config config.Config, posts []Post, url string) (io.Reader, error) { + _, index, err := getPost("content/_index.md") + if err != nil { + return nil, err + } + doc, err := layout("templates/homepage.html", config, config.Title, url) + if err != nil { + return nil, err + } + doc.Find("body").AddClass("h-card") + doc.Find(".title").AddClass("p-name u-url") + + html, err := renderMarkdown(index) + doc.Find("#content").SetHtml(html) + + feed := doc.Find(".h-feed") + tpl := feed.Find(".h-entry") + tpl.Remove() + + for _, post := range posts[0:3] { + entry := tpl.Clone() + entry.Find(".p-name").SetText(post.Title) + entry.Find(".u-url").SetAttr("href", post.URL) + entry. + Find(".dt-published"). + SetAttr("datetime", post.PostMatter.Date.UTC().Format(time.RFC3339)). + SetText(post.PostMatter.Date.Format("2006-01-02")) + + feed.AppendSelection(entry) + } + doc.Find(".u-email"). + SetAttr("href", fmt.Sprintf("mailto:%s", config.Email)). + SetText(config.Email) + + elsewhere := doc.Find(".elsewhere") + linkRelMe := elsewhere.Find(".u-url[rel=me]").ParentsUntil("ul") + linkRelMe.Remove() + + for _, link := range config.Menus["me"] { + el := linkRelMe.Clone() + el.Find("a").SetAttr("href", link.URL).SetText(link.Name) + elsewhere.AppendSelection(el) + } + + return renderHTML(doc), nil +} + +func render404(config config.Config, url string) (io.Reader, error) { + doc, err := layout("templates/404.html", config, "404 Not Found", url) + if err != nil { + return nil, err + } + return renderHTML(doc), nil +} + +func renderFeed(title string, config config.Config, posts []Post, specific string) (io.Reader, error) { + reader, err := loadTemplate("templates/feed.xml") + if err != nil { + return nil, err + } + defer reader.Seek(0, io.SeekStart) + doc, err := xmlquery.Parse(reader) + feed := doc.SelectElement("feed") + feed.SelectElement("title").FirstChild.Data = title + feed.SelectElement("link").SetAttr("href", config.BaseURL.String()) + feed.SelectElement("id").FirstChild.Data = atom.MakeTagURI(config, specific) + datetime, err := posts[0].Date.UTC().MarshalText() + feed.SelectElement("updated").FirstChild.Data = string(datetime) + tpl := feed.SelectElement("entry") + xmlquery.RemoveFromTree(tpl) + + for _, post := range posts { + fullURL, err := url.JoinPath(config.BaseURL.String(), post.URL) + if err != nil { + return nil, err + } + text, err := xml.MarshalIndent(&atom.FeedEntry{ + Title: post.Title, + Link: atom.MakeLink(fullURL), + Id: atom.MakeTagURI(config, post.Basename), + Updated: post.Date.UTC(), + Summary: post.Description, + Author: config.Title, + Content: atom.FeedContent{ + Content: post.Content, + Type: "html", + }, + }, " ", " ") + if err != nil { + return nil, err + } + entry, err := xmlquery.ParseWithOptions(strings.NewReader(string(text)), xmlquery.ParserOptions{ + Decoder: &xmlquery.DecoderOptions{ + Strict: false, + AutoClose: xml.HTMLAutoClose, + Entity: xml.HTMLEntity, + }, + }) + if err != nil { + return nil, err + } + xmlquery.AddChild(feed, entry.SelectElement("entry")) + } + + return strings.NewReader(doc.OutputXML(true)), nil +} + +func renderFeedStyles() (io.Reader, error) { + reader, err := loadTemplate("templates/feed-styles.xsl") + if err != nil { + return nil, err + } + defer reader.Seek(0, io.SeekStart) + nsMap := map[string]string{ + "xsl": "http://www.w3.org/1999/XSL/Transform", + "atom": "http://www.w3.org/2005/Atom", + "xhtml": "http://www.w3.org/1999/xhtml", + } + doc, err := xmlquery.Parse(reader) + expr, err := xpath.CompileWithNS("//xhtml:style", nsMap) + if err != nil { + return nil, err + } + style := xmlquery.QuerySelector(doc, expr) + xmlquery.AddChild(style, &xmlquery.Node{ + Type: xmlquery.TextNode, + Data: string(css), + }) + return strings.NewReader(doc.OutputXML(true)), nil +} + +func renderHTML(doc *goquery.Document) io.Reader { + r, w := io.Pipe() + + // TODO: return errors to main thread + go func() { + w.Write([]byte("\n")) + err := htmlformat.Nodes(w, []*html.Node{doc.Children().Get(0)}) + if err != nil { + slog.Error("error rendering html", "error", err) + w.CloseWithError(err) + return + } + defer w.Close() + }() + return r +} diff --git a/internal/config/config.go b/internal/config/config.go index d2eabf0..578390e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -5,7 +5,6 @@ import ( "log/slog" "net/url" "os" - "strconv" "github.com/BurntSushi/toml" "github.com/pkg/errors" @@ -31,18 +30,17 @@ func (u *URL) UnmarshalText(text []byte) (err error) { } type Config struct { - DefaultLanguage string `toml:"default_language"` - BaseURL URL `toml:"base_url"` - RedirectOtherHostnames bool `toml:"redirect_other_hostnames"` - Port uint64 - Production bool - Title string - Email string - Description string - DomainStartDate string `toml:"domain_start_date"` - OriginalDomain string `toml:"original_domain"` - Taxonomies []Taxonomy - Extra struct { + DefaultLanguage string `toml:"default_language"` + BaseURL URL `toml:"base_url"` + InjectLiveReload bool + Title string + Email string + Description string + DomainStartDate string `toml:"domain_start_date"` + OriginalDomain string `toml:"original_domain"` + Taxonomies []Taxonomy + CSP *CSP `toml:"content-security-policy"` + Extra struct { Headers map[string]string } Menus map[string][]MenuItem @@ -71,10 +69,5 @@ func GetConfig() (*Config, error) { return nil, errors.Wrap(err, "config error") } } - port, err := strconv.ParseUint(getEnvFallback("PORT", "3000"), 10, 16) - if err != nil { - return nil, err - } - config.Port = port return &config, nil } diff --git a/internal/config/csp.go b/internal/config/csp.go new file mode 100644 index 0000000..536d9fc --- /dev/null +++ b/internal/config/csp.go @@ -0,0 +1,44 @@ +// Code generated DO NOT EDIT. +package config + +import ( + "github.com/crewjam/csp" +) + +type CSP struct { + BaseURI []string `csp:"base-uri" toml:"base-uri"` + BlockAllMixedContent bool `csp:"block-all-mixed-content" toml:"block-all-mixed-content"` + ChildSrc []string `csp:"child-src" toml:"child-src"` + ConnectSrc []string `csp:"connect-src" toml:"connect-src"` + DefaultSrc []string `csp:"default-src" toml:"default-src"` + FontSrc []string `csp:"font-src" toml:"font-src"` + FormAction []string `csp:"form-action" toml:"form-action"` + FrameAncestors []string `csp:"frame-ancestors" toml:"frame-ancestors"` + FrameSrc []string `csp:"frame-src" toml:"frame-src"` + ImgSrc []string `csp:"img-src" toml:"img-src"` + ManifestSrc []string `csp:"manifest-src" toml:"manifest-src"` + MediaSrc []string `csp:"media-src" toml:"media-src"` + NavigateTo []string `csp:"navigate-to" toml:"navigate-to"` + ObjectSrc []string `csp:"object-src" toml:"object-src"` + PluginTypes []string `csp:"plugin-types" toml:"plugin-types"` + PrefetchSrc []string `csp:"prefetch-src" toml:"prefetch-src"` + Referrer csp.ReferrerPolicy `csp:"referrer" toml:"referrer"` + ReportTo string `csp:"report-to" toml:"report-to"` + ReportURI string `csp:"report-uri" toml:"report-uri"` + RequireSRIFor []csp.RequireSRIFor `csp:"require-sri-for" toml:"require-sri-for"` + RequireTrustedTypesFor []csp.RequireTrustedTypesFor `csp:"require-trusted-types-for" toml:"require-trusted-types-for"` + Sandbox csp.Sandbox `csp:"sandbox" toml:"sandbox"` + ScriptSrc []string `csp:"script-src" toml:"script-src"` + ScriptSrcAttr []string `csp:"script-src-attr" toml:"script-src-attr"` + ScriptSrcElem []string `csp:"script-src-elem" toml:"script-src-elem"` + StyleSrc []string `csp:"style-src" toml:"style-src"` + StyleSrcAttr []string `csp:"style-src-attr" toml:"style-src-attr"` + StyleSrcElem []string `csp:"style-src-elem" toml:"style-src-elem"` + TrustedTypes []string `csp:"trusted-types" toml:"trusted-types"` + UpgradeInsecureRequests bool `csp:"upgrade-insecure-requests" toml:"upgrade-insecure-requests"` + WorkerSrc []string `csp:"worker-src" toml:"worker-src"` +} + +func (c *CSP) String() string { + return csp.Header(*c).String() +} diff --git a/internal/config/cspgenerator.go b/internal/config/cspgenerator.go new file mode 100644 index 0000000..4594d0d --- /dev/null +++ b/internal/config/cspgenerator.go @@ -0,0 +1,79 @@ +package config + +//go:generate go run ../../cmd/cspgenerator/ + +import ( + "fmt" + "os" + "reflect" + + "github.com/crewjam/csp" + "github.com/fatih/structtag" +) + +func GenerateCSP() error { + t := reflect.TypeFor[csp.Header]() + file, err := os.OpenFile("./csp.go", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0) + if err != nil { + return err + } + defer file.Close() + + _, err = fmt.Fprintf(file, `// Code generated DO NOT EDIT. +package config + +import ( + "github.com/crewjam/csp" +) + +`) + if err != nil { + return err + } + + _, err = fmt.Fprintf(file, "type CSP struct {\n") + if err != nil { + return err + } + + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + var t reflect.Type + if field.Type.Kind() == reflect.Slice { + t = field.Type + } else { + t = field.Type + } + tags, err := structtag.Parse(string(field.Tag)) + if err != nil { + return err + } + cspTag, err := tags.Get("csp") + if err != nil { + return err + } + tags.Set(&structtag.Tag{ + Key: "toml", + Name: cspTag.Name, + }) + + _, err = fmt.Fprintf(file, "\t%-23s %-28s `%s`\n", field.Name, t, tags.String()) + if err != nil { + return err + } + } + _, err = fmt.Fprintln(file, "}") + if err != nil { + return err + } + + _, err = fmt.Fprintln(file, ` +func (c *CSP) String() string { + return csp.Header(*c).String() +} + `) + if err != nil { + return err + } + return nil +} diff --git a/internal/server/filemap.go b/internal/server/filemap.go new file mode 100644 index 0000000..466db49 --- /dev/null +++ b/internal/server/filemap.go @@ -0,0 +1,77 @@ +package server + +import ( + "fmt" + "hash/fnv" + "io" + "io/fs" + "log" + "log/slog" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + +type File struct { + filename string + etag string +} + +var files = map[string]File{} + +func hashFile(filename string) (string, error) { + f, err := os.Open(filename) + if err != nil { + return "", err + } + defer f.Close() + hash := fnv.New64a() + if _, err := io.Copy(hash, f); err != nil { + return "", err + } + return fmt.Sprintf(`W/"%x"`, hash.Sum(nil)), nil +} + +func registerFile(urlpath string, filepath string) error { + if files[urlpath] != (File{}) { + log.Printf("registerFile called with duplicate file, urlPath: %s", urlpath) + return nil + } + hash, err := hashFile(filepath) + if err != nil { + return err + } + files[urlpath] = File{ + filename: filepath, + etag: hash, + } + return nil +} + +func registerContentFiles(root string) error { + err := filepath.WalkDir(root, func(filePath string, f fs.DirEntry, err error) error { + if err != nil { + return errors.WithMessagef(err, "failed to access path %s", filePath) + } + relPath, err := filepath.Rel(root, filePath) + if err != nil { + return errors.WithMessagef(err, "failed to make path relative, path: %s", filePath) + } + urlPath, _ := strings.CutSuffix(relPath, "index.html") + if !f.IsDir() { + slog.Debug("registering file", "urlpath", "/"+urlPath) + return registerFile("/"+urlPath, filePath) + } + return nil + }) + if err != nil { + return err + } + return nil +} + +func GetFile(urlPath string) File { + return files[urlPath] +} diff --git a/internal/server/logging.go b/internal/server/logging.go new file mode 100644 index 0000000..135f06e --- /dev/null +++ b/internal/server/logging.go @@ -0,0 +1,55 @@ +package server + +import ( + "fmt" + "io" + "net/http" +) + +type loggingResponseWriter struct { + http.ResponseWriter + statusCode int +} + +func (lrw *loggingResponseWriter) WriteHeader(code int) { + lrw.statusCode = code + // avoids warning: superfluous response.WriteHeader call + if lrw.statusCode != http.StatusOK { + lrw.ResponseWriter.WriteHeader(code) + } +} + +func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter { + return &loggingResponseWriter{w, http.StatusOK} +} + +type wrappedHandlerOptions struct { + defaultHostname string + logger io.Writer +} + +func wrapHandlerWithLogging(wrappedHandler http.Handler, opts wrappedHandlerOptions) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + scheme := r.Header.Get("X-Forwarded-Proto") + if scheme == "" { + scheme = "http" + } + host := r.Header.Get("Host") + if host == "" { + host = opts.defaultHostname + } + lw := NewLoggingResponseWriter(w) + wrappedHandler.ServeHTTP(lw, r) + statusCode := lw.statusCode + fmt.Fprintf( + opts.logger, + "%s %s %d %s %s %s\n", + scheme, + r.Method, + statusCode, + host, + r.URL.Path, + lw.Header().Get("Location"), + ) + }) +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000..2e9796d --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,221 @@ +package server + +import ( + "context" + "fmt" + "io" + "log" + "log/slog" + "mime" + "net" + "net/http" + "os" + "path" + "slices" + "strings" + "time" + + cfg "website/internal/config" + + "github.com/getsentry/sentry-go" + sentryhttp "github.com/getsentry/sentry-go/http" + "github.com/pkg/errors" + "github.com/shengyanli1982/law" +) + +var config *cfg.Config + +type Config struct { + Production bool `conf:"default:false"` + InDevServer bool `conf:"default:false"` + Root string `conf:"default:website"` + ListenAddress string `conf:"default:localhost"` + Port string `conf:"default:3000,short:p"` + BaseURL cfg.URL `conf:"default:http://localhost:3000,short:b"` +} + +type HTTPError struct { + Error error + Message string + Code int +} + +type Server struct { + *http.Server +} + +func canonicalisePath(path string) (cPath string, differs bool) { + cPath = path + if strings.HasSuffix(path, "/index.html") { + cPath, differs = strings.CutSuffix(path, "index.html") + } else if !strings.HasSuffix(path, "/") && files[path+"/"] != (File{}) { + cPath, differs = path+"/", true + } + return cPath, differs +} + +func serveFile(w http.ResponseWriter, r *http.Request) *HTTPError { + urlPath, shouldRedirect := canonicalisePath(r.URL.Path) + if shouldRedirect { + http.Redirect(w, r, urlPath, 302) + return nil + } + file := GetFile(urlPath) + if file == (File{}) { + return &HTTPError{ + Message: "File not found", + Code: http.StatusNotFound, + } + } + w.Header().Add("ETag", file.etag) + w.Header().Add("Vary", "Accept-Encoding") + w.Header().Add("Content-Security-Policy", config.CSP.String()) + for k, v := range config.Extra.Headers { + w.Header().Add(k, v) + } + + http.ServeFile(w, r, files[urlPath].filename) + return nil +} + +type webHandler func(http.ResponseWriter, *http.Request) *HTTPError + +func (fn webHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + defer func() { + if fail := recover(); fail != nil { + w.WriteHeader(http.StatusInternalServerError) + slog.Error("runtime panic!", "error", fail) + } + }() + w.Header().Set("Server", fmt.Sprintf("website (%s)", ShortSHA)) + if err := fn(w, r); err != nil { + if strings.Contains(r.Header.Get("Accept"), "text/html") { + w.WriteHeader(err.Code) + notFoundPage := "website/private/404.html" + http.ServeFile(w, r, notFoundPage) + } else { + http.Error(w, err.Message, err.Code) + } + } +} + +var newMIMEs = map[string]string{ + ".xsl": "text/xsl", +} + +func fixupMIMETypes() { + for ext, newType := range newMIMEs { + if err := mime.AddExtensionType(ext, newType); err != nil { + slog.Error("could not update mime type", "ext", ext, "mime", newType) + } + } +} + +func applyDevModeOverrides(config *cfg.Config) { + config.CSP.ScriptSrc = slices.Insert(config.CSP.ScriptSrc, 0, "'unsafe-inline'") + config.CSP.ConnectSrc = slices.Insert(config.CSP.ConnectSrc, 0, "'self'") +} + +func New(runtimeConfig *Config) (*Server, error) { + fixupMIMETypes() + + var err error + config, err = cfg.GetConfig() + if err != nil { + return nil, errors.WithMessage(err, "error parsing configuration file") + } + if runtimeConfig.InDevServer { + applyDevModeOverrides(config) + } + + prefix := path.Join(runtimeConfig.Root, "public") + slog.Debug("registering content files", "prefix", prefix) + err = registerContentFiles(prefix) + if err != nil { + return nil, errors.WithMessagef(err, "registering content files") + } + + env := "development" + if runtimeConfig.Production { + env = "production" + } + err = sentry.Init(sentry.ClientOptions{ + EnableTracing: true, + TracesSampleRate: 1.0, + Dsn: os.Getenv("SENTRY_DSN"), + Release: CommitSHA, + Environment: env, + }) + if err != nil { + return nil, errors.WithMessage(err, "could not set up sentry") + } + defer sentry.Flush(2 * time.Second) + sentryHandler := sentryhttp.New(sentryhttp.Options{ + Repanic: true, + }) + + top := http.NewServeMux() + mux := http.NewServeMux() + slog.Debug("binding main handler to", "host", runtimeConfig.BaseURL.Hostname()+"/") + mux.Handle(runtimeConfig.BaseURL.Hostname()+"/", webHandler(serveFile)) + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + newURL := runtimeConfig.BaseURL.String() + r.URL.String() + http.Redirect(w, r, newURL, 301) + }) + + var logWriter io.Writer + if runtimeConfig.Production { + logWriter = law.NewWriteAsyncer(os.Stdout, nil) + } else { + logWriter = os.Stdout + } + top.Handle("/", + sentryHandler.Handle( + wrapHandlerWithLogging(mux, wrappedHandlerOptions{ + defaultHostname: runtimeConfig.BaseURL.Hostname(), + logger: logWriter, + }), + ), + ) + // no logging, no sentry + top.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + listenAddress := net.JoinHostPort(runtimeConfig.ListenAddress, runtimeConfig.Port) + return &Server{ + &http.Server{ + Addr: listenAddress, + Handler: top, + }, + }, nil +} + +func (s *Server) Start() error { + if err := s.ListenAndServe(); err != http.ErrServerClosed { + return err + } + return nil +} + +func (s *Server) Stop() chan struct{} { + slog.Debug("stop called") + + idleConnsClosed := make(chan struct{}) + + go func() { + slog.Debug("shutting down server") + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + err := s.Server.Shutdown(ctx) + slog.Debug("server shut down") + if err != nil { + // Error from closing listeners, or context timeout: + log.Printf("HTTP server Shutdown: %v", err) + } + close(idleConnsClosed) + }() + + return idleConnsClosed +} -- cgit 1.4.1