generate sitemap and robots.txt

3 files changed, 109 insertions, 1 deletions

diff --git a/internal/builder/builder.go b/internal/builder/builder.go
index 0a774fb..aec41e3 100644
--- a/internal/builder/builder.go
+++ b/internal/builder/builder.go
@@ -7,12 +7,14 @@ import (
 	"os"
 	"path"
 	"slices"
+	"time"
 
 	"website/internal/config"
 	"website/internal/log"
 
 	cp "github.com/otiai10/copy"
 	"github.com/pkg/errors"
+	"github.com/snabb/sitemap"
 )
 
 type IOConfig struct {
@@ -43,6 +45,21 @@ func outputToFile(output io.Reader, filename ...string) error {
 	return nil
 }
 
+func writerToFile(writer io.WriterTo, filename ...string) error {
+	log.Debug("outputting file", "filename", path.Join(filename...))
+	file, err := os.OpenFile(path.Join(filename...), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return errors.WithMessage(err, "could not open output file")
+	}
+	defer file.Close()
+
+	if _, err := writer.WriteTo(file); err != nil {
+		return errors.WithMessage(err, "could not write output file")
+	}
+
+	return nil
+}
+
 func build(outDir string, config config.Config) error {
 	log.Debug("output", "dir", outDir)
 	privateDir := path.Join(outDir, "private")
@@ -71,11 +88,21 @@ func build(outDir string, config config.Config) error {
 		return err
 	}
 
+	sm := NewSitemap(config)
+	lastMod := time.Now()
+	if len(posts) > 0 {
+		lastMod = posts[0].Date
+	}
+
 	for _, post := range posts {
 		if err := mkdirp(publicDir, "post", post.Basename); err != nil {
 			return errors.WithMessage(err, "could not create directory for post")
 		}
 		log.Debug("rendering post", "post", post.Basename)
+		sm.Add(&sitemap.URL{
+			Loc:     post.URL,
+			LastMod: &post.Date,
+		})
 		output, err := renderPost(post, config)
 		if err != nil {
 			return errors.WithMessagef(err, "could not render post %s", post.Input)
@@ -96,6 +123,10 @@ func build(outDir string, config config.Config) error {
 	if err := outputToFile(output, publicDir, "tags", "index.html"); err != nil {
 		return err
 	}
+	sm.Add(&sitemap.URL{
+		Loc:     "/tags/",
+		LastMod: &lastMod,
+	})
 
 	for _, tag := range tags.ToSlice() {
 		matchingPosts := []Post{}
@@ -108,13 +139,18 @@ func build(outDir string, config config.Config) error {
 			return errors.WithMessage(err, "could not create directory")
 		}
 		log.Debug("rendering tags page", "tag", tag)
-		output, err := renderListPage(tag, config, matchingPosts, "/tags/"+tag)
+		url := "/tags/" + tag
+		output, err := renderListPage(tag, config, matchingPosts, url)
 		if err != nil {
 			return errors.WithMessage(err, "could not render tag page")
 		}
 		if err := outputToFile(output, publicDir, "tags", tag, "index.html"); err != nil {
 			return err
 		}
+		sm.Add(&sitemap.URL{
+			Loc:     url,
+			LastMod: &matchingPosts[0].Date,
+		})
 
 		log.Debug("rendering tags feed", "tag", tag)
 		output, err = renderFeed(
@@ -139,6 +175,10 @@ func build(outDir string, config config.Config) error {
 	if err := outputToFile(listPage, publicDir, "post", "index.html"); err != nil {
 		return err
 	}
+	sm.Add(&sitemap.URL{
+		Loc:     "/post/",
+		LastMod: &lastMod,
+	})
 
 	log.Debug("rendering feed")
 	feed, err := renderFeed(config.Title, config, posts, "feed")
@@ -166,6 +206,12 @@ func build(outDir string, config config.Config) error {
 	if err := outputToFile(homePage, publicDir, "index.html"); err != nil {
 		return err
 	}
+	// it would be nice to set LastMod here, but using the latest post
+	// date would be wrong as the homepage has its own content file
+	// without a date, which could be newer
+	sm.Add(&sitemap.URL{
+		Loc: "/",
+	})
 
 	log.Debug("rendering 404 page")
 	notFound, err := render404(config, "/404.html")
@@ -176,6 +222,20 @@ func build(outDir string, config config.Config) error {
 		return err
 	}
 
+	log.Debug("rendering sitemap")
+	if err := writerToFile(sm, publicDir, "sitemap.xml"); err != nil {
+		return err
+	}
+
+	log.Debug("rendering robots.txt")
+	rob, err := renderRobotsTXT(config)
+	if err != nil {
+		return err
+	}
+	if err := outputToFile(rob, publicDir, "robots.txt"); err != nil {
+		return err
+	}
+
 	return nil
 }
 
diff --git a/internal/builder/sitemap.go b/internal/builder/sitemap.go
new file mode 100644
index 0000000..81e3a31
--- /dev/null
+++ b/internal/builder/sitemap.go
@@ -0,0 +1,29 @@
+package builder
+
+import (
+	"io"
+	"website/internal/config"
+
+	"github.com/snabb/sitemap"
+)
+
+type Sitemap struct {
+	config  *config.Config
+	Sitemap *sitemap.Sitemap
+}
+
+func NewSitemap(cfg config.Config) *Sitemap {
+	return &Sitemap{
+		config:  &cfg,
+		Sitemap: sitemap.New(),
+	}
+}
+
+func (s *Sitemap) Add(u *sitemap.URL) {
+	u.Loc = s.config.BaseURL.JoinPath(u.Loc).String()
+	s.Sitemap.Add(u)
+}
+
+func (s *Sitemap) WriteTo(w io.Writer) (int64, error) {
+	return s.Sitemap.WriteTo(w)
+}
diff --git a/internal/builder/template.go b/internal/builder/template.go
index 5e2af1a..ab36c85 100644
--- a/internal/builder/template.go
+++ b/internal/builder/template.go
@@ -8,6 +8,7 @@ import (
 	"os"
 	"strings"
 	"sync"
+	"text/template"
 	"time"
 	"website/internal/atom"
 	"website/internal/config"
@@ -295,6 +296,24 @@ func renderHomepage(config config.Config, posts []Post, url string) (io.Reader,
 	return renderHTML(doc), nil
 }
 
+func renderRobotsTXT(config config.Config) (io.Reader, error) {
+	r, w := io.Pipe()
+	tpl, err := template.ParseFiles("templates/robots.tmpl")
+	if err != nil {
+		return nil, err
+	}
+	go func() {
+		err = tpl.Execute(w, map[string]interface{}{
+			"BaseURL": config.BaseURL,
+		})
+		if err != nil {
+			w.CloseWithError(err)
+		}
+		w.Close()
+	}()
+	return r, nil
+}
+
 func render404(config config.Config, url string) (io.Reader, error) {
 	doc, err := layout("templates/404.html", config, "404 Not Found", url)
 	if err != nil {