From 2a4c795d5a165f995e9f7dc84e07465b140f3770 Mon Sep 17 00:00:00 2001 From: Alan Pearce Date: Sat, 27 Apr 2024 21:18:03 +0200 Subject: implement live-reloading dev server Squashed commit of the following: commit 02f077432202af4d633eb2cad81dfdaa6921317f Author: Alan Pearce Date: Sat Apr 27 21:09:14 2024 +0200 builder: only remove output directory if set and in dev mode commit 47001e01c55fa6e74aafeda04ebc3e4e7c47eba0 Author: Alan Pearce Date: Sat Apr 27 21:03:37 2024 +0200 implement live reload on dev server commit 411ec969f61e4b73439f1c54ea29f75135ecc618 Author: Alan Pearce Date: Sat Apr 27 20:59:26 2024 +0200 server: implement graceful shutdown commit 5400132eb6eb1b638e0b3fd4265f51611c92d473 Author: Alan Pearce Date: Sat Apr 27 20:41:07 2024 +0200 add some debug logs commit 3c9b678197c044603950232d222f501ef74d7873 Author: Alan Pearce Date: Sat Apr 27 20:39:09 2024 +0200 prefix log output with executable name commit 300e24c179e390e9d3f5aeab4471c97f17f1fa64 Author: Alan Pearce Date: Sat Apr 27 20:29:42 2024 +0200 don't panic inside internal packages, return error instead commit fe2715d330402ad67fe866471bed89c7238ad2ec Author: Alan Pearce Date: Fri Apr 26 01:18:29 2024 +0200 config: use a table to configure CSP headers commit d012553aaf78a436fa8871830b5d720a9e292d4b Author: Alan Pearce Date: Thu Apr 25 17:13:39 2024 +0200 dev: create basic dev server to build and serve from a temporary directory commit a1d11d3e69650d9b43ca1b1d7b7ccc05a808d5c1 Author: Alan Pearce Date: Thu Apr 25 13:02:22 2024 +0200 remove unused redirect_other_hostnames config option commit fd67b19b5c7f76f0c3579e8a05ef20a618e90be7 Author: Alan Pearce Date: Thu Apr 25 12:58:53 2024 +0200 server: make port a string, which is what go uses commit c798e8e736c0649008cade337158399470a9099b Author: Alan Pearce Date: Thu Apr 25 12:58:33 2024 +0200 config: remove unused port variable commit f94882b9001f3b0855e26b26b4a84b96e3deb22b Author: Alan Pearce Date: Thu Apr 25 12:49:10 2024 +0200 re-organise module layout --- cmd/build/build.go | 196 --------------------------- cmd/build/main.go | 8 +- cmd/build/posts.go | 121 ----------------- cmd/build/template.go | 359 -------------------------------------------------- 4 files changed, 6 insertions(+), 678 deletions(-) delete mode 100644 cmd/build/build.go delete mode 100644 cmd/build/posts.go delete mode 100644 cmd/build/template.go (limited to 'cmd/build') diff --git a/cmd/build/build.go b/cmd/build/build.go deleted file mode 100644 index 5daa940..0000000 --- a/cmd/build/build.go +++ /dev/null @@ -1,196 +0,0 @@ -package main - -import ( - "fmt" - "io" - "log" - "log/slog" - "net/url" - "os" - "path" - "slices" - - "website/internal/config" - - cp "github.com/otiai10/copy" - "github.com/pkg/errors" -) - -func mkdirp(dirs ...string) error { - return os.MkdirAll(path.Join(dirs...), 0755) -} - -func outputToFile(output io.Reader, filename ...string) error { - file, err := os.OpenFile(path.Join(filename...), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return errors.WithMessage(err, "could not open output file") - } - defer file.Close() - - if _, err := file.ReadFrom(output); err != nil { - return errors.WithMessage(err, "could not write output file") - } - return nil -} - -func build(outDir string, config config.Config) error { - privateDir := path.Join(outDir, "private") - if err := mkdirp(privateDir); err != nil { - return errors.WithMessage(err, "could not create private directory") - } - publicDir := path.Join(outDir, "public") - if err := mkdirp(publicDir); err != nil { - return errors.WithMessage(err, "could not create public directory") - } - - err := cp.Copy("static", publicDir, cp.Options{ - PreserveTimes: true, - PermissionControl: cp.AddPermission(0755), - }) - if err != nil { - log.Panic(errors.Errorf("could not copy static files: %v", err)) - } - - if err := mkdirp(publicDir, "post"); err != nil { - return errors.WithMessage(err, "could not create post output directory") - } - slog.Debug("reading posts") - posts, tags, err := readPosts("content", "post", publicDir) - if err != nil { - return err - } - - for _, post := range posts { - if err := mkdirp(publicDir, "post", post.Basename); err != nil { - return errors.WithMessage(err, "could not create directory for post") - } - slog.Debug("rendering post", "post", post.Basename) - output, err := renderPost(post, config) - if err != nil { - return errors.WithMessagef(err, "could not render post %s", post.Input) - } - if err := outputToFile(output, post.Output); err != nil { - return err - } - } - - if err := mkdirp(publicDir, "tags"); err != nil { - return errors.WithMessage(err, "could not create directory for tags") - } - slog.Debug("rendering tags list") - output, err := renderTags(tags, config, "/tags") - if err != nil { - return errors.WithMessage(err, "could not render tags") - } - if err := outputToFile(output, publicDir, "tags", "index.html"); err != nil { - return err - } - - for _, tag := range tags.ToSlice() { - matchingPosts := []Post{} - for _, post := range posts { - if slices.Contains(post.Taxonomies.Tags, tag) { - matchingPosts = append(matchingPosts, post) - } - } - if err := mkdirp(publicDir, "tags", tag); err != nil { - return errors.WithMessage(err, "could not create directory") - } - slog.Debug("rendering tags page", "tag", tag) - output, err := renderListPage(tag, config, matchingPosts, "/tags/"+tag) - if err != nil { - return errors.WithMessage(err, "could not render tag page") - } - if err := outputToFile(output, publicDir, "tags", tag, "index.html"); err != nil { - return err - } - - slog.Debug("rendering tags feed", "tag", tag) - output, err = renderFeed(fmt.Sprintf("%s - %s", config.Title, tag), config, matchingPosts, tag) - if err != nil { - return errors.WithMessage(err, "could not render tag feed page") - } - if err := outputToFile(output, publicDir, "tags", tag, "atom.xml"); err != nil { - return err - } - } - - slog.Debug("rendering list page") - listPage, err := renderListPage("", config, posts, "/post") - if err != nil { - return errors.WithMessage(err, "could not render list page") - } - if err := outputToFile(listPage, publicDir, "post", "index.html"); err != nil { - return err - } - - slog.Debug("rendering feed") - feed, err := renderFeed(config.Title, config, posts, "feed") - if err != nil { - return errors.WithMessage(err, "could not render feed") - } - if err := outputToFile(feed, publicDir, "atom.xml"); err != nil { - return err - } - - slog.Debug("rendering feed styles") - feedStyles, err := renderFeedStyles() - if err != nil { - return errors.WithMessage(err, "could not render feed styles") - } - if err := outputToFile(feedStyles, publicDir, "feed-styles.xsl"); err != nil { - return err - } - - slog.Debug("rendering homepage") - homePage, err := renderHomepage(config, posts, "/") - if err != nil { - return errors.WithMessage(err, "could not render homepage") - } - if err := outputToFile(homePage, publicDir, "index.html"); err != nil { - return err - } - - slog.Debug("rendering 404 page") - notFound, err := render404(config, "/404.html") - if err != nil { - return errors.WithMessage(err, "could not render 404 page") - } - if err := outputToFile(notFound, privateDir, "404.html"); err != nil { - return err - } - - return nil -} - -type IOConfig struct { - Source string `conf:"default:.,short:s"` - Destination string `conf:"default:website,short:d"` - BaseURL config.URL -} - -func buildSite(ioConfig IOConfig) error { - config, err := config.GetConfig() - if err != nil { - log.Panic(errors.Errorf("could not get config: %v", err)) - } - - if ioConfig.BaseURL.URL != nil { - config.BaseURL.URL, err = url.Parse(ioConfig.BaseURL.String()) - if err != nil { - log.Panic(errors.Errorf("highly unlikely: %v", err)) - } - } - - err = os.RemoveAll(ioConfig.Destination) - if err != nil { - log.Panic(errors.Errorf("could not remove public directory: %v", err)) - } - - if err := build(ioConfig.Destination, *config); err != nil { - return err - } - - slog.Debug("done") - return nil -} diff --git a/cmd/build/main.go b/cmd/build/main.go index 9b6a79b..069f9bd 100644 --- a/cmd/build/main.go +++ b/cmd/build/main.go @@ -7,6 +7,8 @@ import ( "log/slog" "os" + "website/internal/builder" + "github.com/BurntSushi/toml" "github.com/ardanlabs/conf/v3" "github.com/pkg/errors" @@ -16,9 +18,11 @@ func main() { if os.Getenv("DEBUG") != "" { slog.SetLogLoggerLevel(slog.LevelDebug) } + log.SetFlags(log.LstdFlags | log.Lmsgprefix) + log.SetPrefix("build: ") slog.Debug("starting build process") - ioConfig := IOConfig{} + ioConfig := builder.IOConfig{} if help, err := conf.Parse("", &ioConfig); err != nil { if errors.Is(err, conf.ErrHelpWanted) { fmt.Println(help) @@ -34,7 +38,7 @@ func main() { } } - if err := buildSite(ioConfig); err != nil { + if err := builder.BuildSite(ioConfig); err != nil { switch cause := errors.Cause(err).(type) { case *fs.PathError: slog.Info("pathError") diff --git a/cmd/build/posts.go b/cmd/build/posts.go deleted file mode 100644 index f03caf3..0000000 --- a/cmd/build/posts.go +++ /dev/null @@ -1,121 +0,0 @@ -package main - -import ( - "bytes" - "log/slog" - "os" - "path" - "path/filepath" - "slices" - "strings" - "time" - - "github.com/adrg/frontmatter" - mapset "github.com/deckarep/golang-set/v2" - "github.com/pkg/errors" - "github.com/yuin/goldmark" - "github.com/yuin/goldmark/extension" - htmlrenderer "github.com/yuin/goldmark/renderer/html" -) - -type PostMatter struct { - Date time.Time `toml:"date"` - Description string `toml:"description"` - Title string `toml:"title"` - Taxonomies struct { - Tags []string `toml:"tags"` - } `toml:"taxonomies"` -} - -type Post struct { - Input string - Output string - Basename string - URL string - Content string - PostMatter -} - -type Tags mapset.Set[string] - -var markdown = goldmark.New( - goldmark.WithRendererOptions( - htmlrenderer.WithUnsafe(), - ), - goldmark.WithExtensions( - extension.GFM, - extension.Footnote, - extension.Typographer, - ), -) - -func getPost(filename string) (*PostMatter, []byte, error) { - matter := PostMatter{} - content, err := os.Open(filename) - defer content.Close() - if err != nil { - return nil, nil, errors.WithMessagef(err, "could not open post %s", filename) - } - rest, err := frontmatter.MustParse(content, &matter) - if err != nil { - return nil, nil, errors.WithMessagef(err, "could not parse front matter of post %s", filename) - } - - return &matter, rest, nil -} - -func renderMarkdown(content []byte) (string, error) { - var buf bytes.Buffer - if err := markdown.Convert(content, &buf); err != nil { - return "", errors.WithMessage(err, "could not convert markdown content") - } - return buf.String(), nil -} - -func readPosts(root string, inputDir string, outputDir string) ([]Post, Tags, error) { - tags := mapset.NewSet[string]() - posts := []Post{} - subdir := filepath.Join(root, inputDir) - files, err := os.ReadDir(subdir) - if err != nil { - return nil, nil, errors.WithMessagef(err, "could not read post directory %s", subdir) - } - outputReplacer := strings.NewReplacer(root, outputDir, ".md", "/index.html") - urlReplacer := strings.NewReplacer(root, "", ".md", "/") - for _, f := range files { - pathFromRoot := filepath.Join(subdir, f.Name()) - if !f.IsDir() && path.Ext(pathFromRoot) == ".md" { - output := outputReplacer.Replace(pathFromRoot) - url := urlReplacer.Replace(pathFromRoot) - slog.Debug("reading post", "post", pathFromRoot) - matter, content, err := getPost(pathFromRoot) - if err != nil { - return nil, nil, err - } - - for _, tag := range matter.Taxonomies.Tags { - tags.Add(strings.ToLower(tag)) - } - - slog.Debug("rendering markdown in post", "post", pathFromRoot) - html, err := renderMarkdown(content) - if err != nil { - return nil, nil, err - } - post := Post{ - Input: pathFromRoot, - Output: output, - Basename: filepath.Base(url), - URL: url, - PostMatter: *matter, - Content: html, - } - - posts = append(posts, post) - } - } - slices.SortFunc(posts, func(a, b Post) int { - return b.Date.Compare(a.Date) - }) - return posts, tags, nil -} diff --git a/cmd/build/template.go b/cmd/build/template.go deleted file mode 100644 index 84cf57c..0000000 --- a/cmd/build/template.go +++ /dev/null @@ -1,359 +0,0 @@ -package main - -import ( - "encoding/xml" - "fmt" - "io" - "log/slog" - "net/url" - "os" - "strings" - "sync" - "time" - "website/internal/atom" - "website/internal/config" - - "github.com/PuerkitoBio/goquery" - "github.com/a-h/htmlformat" - "github.com/antchfx/xmlquery" - "github.com/antchfx/xpath" - mapset "github.com/deckarep/golang-set/v2" - "golang.org/x/net/html" -) - -var ( - assetsOnce sync.Once - css string - countHTML *goquery.Document - templates map[string]*os.File = make(map[string]*os.File) -) - -func loadTemplate(path string) (file *os.File, err error) { - if templates[path] == nil { - file, err = os.OpenFile(path, os.O_RDONLY, 0) - if err != nil { - return nil, err - } - templates[path] = file - } - file = templates[path] - return -} - -var ( - imgOnce sync.Once - img *goquery.Selection - urlTemplate *url.URL -) - -type QuerySelection struct { - *goquery.Selection -} - -type QueryDocument struct { - *goquery.Document -} - -func NewDocumentFromReader(r io.Reader) (*QueryDocument, error) { - doc, err := goquery.NewDocumentFromReader(r) - return &QueryDocument{doc}, err -} - -func (q *QueryDocument) Find(selector string) *QuerySelection { - return &QuerySelection{q.Document.Find(selector)} -} - -func NewDocumentNoScript(r io.Reader) (*goquery.Document, error) { - root, err := html.ParseWithOptions(r, html.ParseOptionEnableScripting(false)) - return goquery.NewDocumentFromNode(root), err -} - -func (root QuerySelection) setImgURL(pageURL string, pageTitle string) QuerySelection { - clone := countHTML.Clone() - imgOnce.Do(func() { - var err error - img = clone.Find("img") - attr, _ := img.Attr("src") - if attr == "" { - panic(" does not have src attribute") - } - urlTemplate, err = url.Parse(attr) - if err != nil { - panic(err.Error()) - } - }) - q := urlTemplate.Query() - urlTemplate.RawQuery = "" - q.Set("p", pageURL) - q.Set("t", pageTitle) - output := urlTemplate.String() + "?" + q.Encode() - clone.Find("img").SetAttr("src", output) - root.AppendSelection(clone.Find("body").Children()) - return root -} - -func layout(filename string, config config.Config, pageTitle string, pageURL string) (*goquery.Document, error) { - html, err := loadTemplate(filename) - if err != nil { - return nil, err - } - defer html.Seek(0, io.SeekStart) - assetsOnce.Do(func() { - var bytes []byte - bytes, err = os.ReadFile("templates/style.css") - if err != nil { - return - } - css = string(bytes) - countFile, err := os.OpenFile("templates/count.html", os.O_RDONLY, 0) - if err != nil { - return - } - defer countFile.Close() - countHTML, err = NewDocumentNoScript(countFile) - }) - if err != nil { - return nil, err - } - - doc, err := NewDocumentFromReader(html) - if err != nil { - return nil, err - } - doc.Find("html").SetAttr("lang", config.DefaultLanguage) - doc.Find("head > link[rel=alternate]").SetAttr("title", config.Title) - doc.Find("head > link[rel=canonical]").SetAttr("href", pageURL) - doc.Find(".title").SetText(config.Title) - doc.Find("title").Add(".p-name").SetText(pageTitle) - doc.Find("head > style").SetHtml(css) - doc.Find("body").setImgURL(pageURL, pageTitle) - nav := doc.Find("nav") - navLink := doc.Find("nav a") - nav.Empty() - for _, link := range config.Menus["main"] { - nav.AppendSelection(navLink.Clone().SetAttr("href", link.URL).SetText(link.Name)) - } - return doc.Document, nil -} - -func renderPost(post Post, config config.Config) (r io.Reader, err error) { - doc, err := layout("templates/post.html", config, post.PostMatter.Title, post.URL) - if err != nil { - return nil, err - } - doc.Find(".title").AddClass("p-author h-card").SetAttr("rel", "author") - doc.Find(".h-entry .dt-published"). - SetAttr("datetime", post.PostMatter.Date.UTC().Format(time.RFC3339)). - SetText( - post.PostMatter.Date.Format("2006-01-02"), - ) - doc.Find(".h-entry .e-content").SetHtml(post.Content) - categories := doc.Find(".h-entry .p-categories") - tpl := categories.Find(".p-category").ParentsUntilSelection(categories) - tpl.Remove() - for _, tag := range post.Taxonomies.Tags { - cat := tpl.Clone() - cat.Find(".p-category").SetAttr("href", fmt.Sprintf("/tags/%s/", tag)).SetText("#" + tag) - categories.AppendSelection(cat) - } - - return renderHTML(doc), nil -} - -func renderTags(tags Tags, config config.Config, url string) (io.Reader, error) { - doc, err := layout("templates/tags.html", config, config.Title, url) - if err != nil { - return nil, err - } - tagList := doc.Find(".tags") - tpl := doc.Find(".h-feed") - tpl.Remove() - for _, tag := range mapset.Sorted(tags) { - li := tpl.Clone() - li.Find("a").SetAttr("href", fmt.Sprintf("/tags/%s/", tag)).SetText("#" + tag) - tagList.AppendSelection(li) - } - return renderHTML(doc), nil -} - -func renderListPage(tag string, config config.Config, posts []Post, url string) (io.Reader, error) { - var title string - if len(tag) > 0 { - title = tag - } else { - title = config.Title - } - doc, err := layout("templates/list.html", config, title, url) - if err != nil { - return nil, err - } - feed := doc.Find(".h-feed") - tpl := feed.Find(".h-entry") - tpl.Remove() - - doc.Find(".title").AddClass("p-author h-card").SetAttr("rel", "author") - if tag == "" { - doc.Find(".filter").Remove() - } else { - doc.Find(".filter").Find("h3").SetText("#" + tag) - } - - for _, post := range posts { - entry := tpl.Clone() - entry.Find(".p-name").SetText(post.Title).SetAttr("href", post.URL) - entry.Find(".dt-published"). - SetAttr("datetime", post.PostMatter.Date.UTC().Format(time.RFC3339)). - SetText(post.PostMatter.Date.Format("2006-01-02")) - feed.AppendSelection(entry) - } - - return renderHTML(doc), nil -} - -func renderHomepage(config config.Config, posts []Post, url string) (io.Reader, error) { - _, index, err := getPost("content/_index.md") - if err != nil { - return nil, err - } - doc, err := layout("templates/homepage.html", config, config.Title, url) - if err != nil { - return nil, err - } - doc.Find("body").AddClass("h-card") - doc.Find(".title").AddClass("p-name u-url") - - html, err := renderMarkdown(index) - doc.Find("#content").SetHtml(html) - - feed := doc.Find(".h-feed") - tpl := feed.Find(".h-entry") - tpl.Remove() - - for _, post := range posts[0:3] { - entry := tpl.Clone() - entry.Find(".p-name").SetText(post.Title) - entry.Find(".u-url").SetAttr("href", post.URL) - entry. - Find(".dt-published"). - SetAttr("datetime", post.PostMatter.Date.UTC().Format(time.RFC3339)). - SetText(post.PostMatter.Date.Format("2006-01-02")) - - feed.AppendSelection(entry) - } - doc.Find(".u-email"). - SetAttr("href", fmt.Sprintf("mailto:%s", config.Email)). - SetText(config.Email) - - elsewhere := doc.Find(".elsewhere") - linkRelMe := elsewhere.Find(".u-url[rel=me]").ParentsUntil("ul") - linkRelMe.Remove() - - for _, link := range config.Menus["me"] { - el := linkRelMe.Clone() - el.Find("a").SetAttr("href", link.URL).SetText(link.Name) - elsewhere.AppendSelection(el) - } - - return renderHTML(doc), nil -} - -func render404(config config.Config, url string) (io.Reader, error) { - doc, err := layout("templates/404.html", config, "404 Not Found", url) - if err != nil { - return nil, err - } - return renderHTML(doc), nil -} - -func renderFeed(title string, config config.Config, posts []Post, specific string) (io.Reader, error) { - reader, err := loadTemplate("templates/feed.xml") - if err != nil { - return nil, err - } - defer reader.Seek(0, io.SeekStart) - doc, err := xmlquery.Parse(reader) - feed := doc.SelectElement("feed") - feed.SelectElement("title").FirstChild.Data = title - feed.SelectElement("link").SetAttr("href", config.BaseURL.String()) - feed.SelectElement("id").FirstChild.Data = atom.MakeTagURI(config, specific) - datetime, err := posts[0].Date.UTC().MarshalText() - feed.SelectElement("updated").FirstChild.Data = string(datetime) - tpl := feed.SelectElement("entry") - xmlquery.RemoveFromTree(tpl) - - for _, post := range posts { - fullURL, err := url.JoinPath(config.BaseURL.String(), post.URL) - if err != nil { - return nil, err - } - text, err := xml.MarshalIndent(&atom.FeedEntry{ - Title: post.Title, - Link: atom.MakeLink(fullURL), - Id: atom.MakeTagURI(config, post.Basename), - Updated: post.Date.UTC(), - Summary: post.Description, - Author: config.Title, - Content: atom.FeedContent{ - Content: post.Content, - Type: "html", - }, - }, " ", " ") - if err != nil { - return nil, err - } - entry, err := xmlquery.ParseWithOptions(strings.NewReader(string(text)), xmlquery.ParserOptions{ - Decoder: &xmlquery.DecoderOptions{ - Strict: false, - AutoClose: xml.HTMLAutoClose, - Entity: xml.HTMLEntity, - }, - }) - if err != nil { - return nil, err - } - xmlquery.AddChild(feed, entry.SelectElement("entry")) - } - - return strings.NewReader(doc.OutputXML(true)), nil -} - -func renderFeedStyles() (io.Reader, error) { - reader, err := loadTemplate("templates/feed-styles.xsl") - if err != nil { - return nil, err - } - defer reader.Seek(0, io.SeekStart) - nsMap := map[string]string{ - "xsl": "http://www.w3.org/1999/XSL/Transform", - "atom": "http://www.w3.org/2005/Atom", - "xhtml": "http://www.w3.org/1999/xhtml", - } - doc, err := xmlquery.Parse(reader) - expr, err := xpath.CompileWithNS("//xhtml:style", nsMap) - if err != nil { - return nil, err - } - style := xmlquery.QuerySelector(doc, expr) - xmlquery.AddChild(style, &xmlquery.Node{ - Type: xmlquery.TextNode, - Data: string(css), - }) - return strings.NewReader(doc.OutputXML(true)), nil -} - -func renderHTML(doc *goquery.Document) io.Reader { - r, w := io.Pipe() - - // TODO: return errors to main thread - go func() { - w.Write([]byte("\n")) - err := htmlformat.Nodes(w, []*html.Node{doc.Children().Get(0)}) - if err != nil { - slog.Error("error rendering html", "error", err) - w.CloseWithError(err) - return - } - defer w.Close() - }() - return r -} -- cgit 1.4.1