From a9337d346ca6d82bbe203d50b176af9b7c146db0 Mon Sep 17 00:00:00 2001 From: Alan Pearce Date: Wed, 5 Jun 2024 22:15:05 +0200 Subject: generate sitemap and robots.txt --- internal/builder/builder.go | 62 +++++++++++++++++++++++++++++++++++++++++++- internal/builder/sitemap.go | 29 +++++++++++++++++++++ internal/builder/template.go | 19 ++++++++++++++ 3 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 internal/builder/sitemap.go (limited to 'internal') diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 0a774fb..aec41e3 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -7,12 +7,14 @@ import ( "os" "path" "slices" + "time" "website/internal/config" "website/internal/log" cp "github.com/otiai10/copy" "github.com/pkg/errors" + "github.com/snabb/sitemap" ) type IOConfig struct { @@ -43,6 +45,21 @@ func outputToFile(output io.Reader, filename ...string) error { return nil } +func writerToFile(writer io.WriterTo, filename ...string) error { + log.Debug("outputting file", "filename", path.Join(filename...)) + file, err := os.OpenFile(path.Join(filename...), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return errors.WithMessage(err, "could not open output file") + } + defer file.Close() + + if _, err := writer.WriteTo(file); err != nil { + return errors.WithMessage(err, "could not write output file") + } + + return nil +} + func build(outDir string, config config.Config) error { log.Debug("output", "dir", outDir) privateDir := path.Join(outDir, "private") @@ -71,11 +88,21 @@ func build(outDir string, config config.Config) error { return err } + sm := NewSitemap(config) + lastMod := time.Now() + if len(posts) > 0 { + lastMod = posts[0].Date + } + for _, post := range posts { if err := mkdirp(publicDir, "post", post.Basename); err != nil { return errors.WithMessage(err, "could not create directory for post") } log.Debug("rendering post", "post", post.Basename) + sm.Add(&sitemap.URL{ + Loc: post.URL, + LastMod: &post.Date, + }) output, err := renderPost(post, config) if err != nil { return errors.WithMessagef(err, "could not render post %s", post.Input) @@ -96,6 +123,10 @@ func build(outDir string, config config.Config) error { if err := outputToFile(output, publicDir, "tags", "index.html"); err != nil { return err } + sm.Add(&sitemap.URL{ + Loc: "/tags/", + LastMod: &lastMod, + }) for _, tag := range tags.ToSlice() { matchingPosts := []Post{} @@ -108,13 +139,18 @@ func build(outDir string, config config.Config) error { return errors.WithMessage(err, "could not create directory") } log.Debug("rendering tags page", "tag", tag) - output, err := renderListPage(tag, config, matchingPosts, "/tags/"+tag) + url := "/tags/" + tag + output, err := renderListPage(tag, config, matchingPosts, url) if err != nil { return errors.WithMessage(err, "could not render tag page") } if err := outputToFile(output, publicDir, "tags", tag, "index.html"); err != nil { return err } + sm.Add(&sitemap.URL{ + Loc: url, + LastMod: &matchingPosts[0].Date, + }) log.Debug("rendering tags feed", "tag", tag) output, err = renderFeed( @@ -139,6 +175,10 @@ func build(outDir string, config config.Config) error { if err := outputToFile(listPage, publicDir, "post", "index.html"); err != nil { return err } + sm.Add(&sitemap.URL{ + Loc: "/post/", + LastMod: &lastMod, + }) log.Debug("rendering feed") feed, err := renderFeed(config.Title, config, posts, "feed") @@ -166,6 +206,12 @@ func build(outDir string, config config.Config) error { if err := outputToFile(homePage, publicDir, "index.html"); err != nil { return err } + // it would be nice to set LastMod here, but using the latest post + // date would be wrong as the homepage has its own content file + // without a date, which could be newer + sm.Add(&sitemap.URL{ + Loc: "/", + }) log.Debug("rendering 404 page") notFound, err := render404(config, "/404.html") @@ -176,6 +222,20 @@ func build(outDir string, config config.Config) error { return err } + log.Debug("rendering sitemap") + if err := writerToFile(sm, publicDir, "sitemap.xml"); err != nil { + return err + } + + log.Debug("rendering robots.txt") + rob, err := renderRobotsTXT(config) + if err != nil { + return err + } + if err := outputToFile(rob, publicDir, "robots.txt"); err != nil { + return err + } + return nil } diff --git a/internal/builder/sitemap.go b/internal/builder/sitemap.go new file mode 100644 index 0000000..81e3a31 --- /dev/null +++ b/internal/builder/sitemap.go @@ -0,0 +1,29 @@ +package builder + +import ( + "io" + "website/internal/config" + + "github.com/snabb/sitemap" +) + +type Sitemap struct { + config *config.Config + Sitemap *sitemap.Sitemap +} + +func NewSitemap(cfg config.Config) *Sitemap { + return &Sitemap{ + config: &cfg, + Sitemap: sitemap.New(), + } +} + +func (s *Sitemap) Add(u *sitemap.URL) { + u.Loc = s.config.BaseURL.JoinPath(u.Loc).String() + s.Sitemap.Add(u) +} + +func (s *Sitemap) WriteTo(w io.Writer) (int64, error) { + return s.Sitemap.WriteTo(w) +} diff --git a/internal/builder/template.go b/internal/builder/template.go index 5e2af1a..ab36c85 100644 --- a/internal/builder/template.go +++ b/internal/builder/template.go @@ -8,6 +8,7 @@ import ( "os" "strings" "sync" + "text/template" "time" "website/internal/atom" "website/internal/config" @@ -295,6 +296,24 @@ func renderHomepage(config config.Config, posts []Post, url string) (io.Reader, return renderHTML(doc), nil } +func renderRobotsTXT(config config.Config) (io.Reader, error) { + r, w := io.Pipe() + tpl, err := template.ParseFiles("templates/robots.tmpl") + if err != nil { + return nil, err + } + go func() { + err = tpl.Execute(w, map[string]interface{}{ + "BaseURL": config.BaseURL, + }) + if err != nil { + w.CloseWithError(err) + } + w.Close() + }() + return r, nil +} + func render404(config config.Config, url string) (io.Reader, error) { doc, err := layout("templates/404.html", config, "404 Not Found", url) if err != nil { -- cgit 1.4.1