From 6c4a3268bc4c528ecff45f50ed5ca6aa1d48500c Mon Sep 17 00:00:00 2001 From: Alan Pearce Date: Fri, 28 Jun 2024 20:43:14 +0200 Subject: wip --- cmd/postdate/main.go | 71 +++++++++++++++++++++++++++++++++++ content/dates.json | 54 +++++++++++++++++++++++++++ internal/builder/builder.go | 18 ++++++--- internal/content/posts.go | 91 +++++++++++++++++++++++++++------------------ internal/server/server.go | 8 +++- internal/vcs/repository.go | 26 ++++++++++--- internal/website/filemap.go | 4 +- 7 files changed, 222 insertions(+), 50 deletions(-) create mode 100644 cmd/postdate/main.go create mode 100644 content/dates.json diff --git a/cmd/postdate/main.go b/cmd/postdate/main.go new file mode 100644 index 0000000..2f340cf --- /dev/null +++ b/cmd/postdate/main.go @@ -0,0 +1,71 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "sync" + "website/internal/builder" + "website/internal/content" + "website/internal/log" + "website/internal/vcs" + + "github.com/go-git/go-git/v5/plumbing/object" +) + +func getPostVCSDates(iter object.CommitIter) *content.PostVCSDates { + p := &content.PostVCSDates{} + defer iter.Close() + for c, err := iter.Next(); c != nil && err == nil; c, err = iter.Next() { + if p.LastUpdated.IsZero() { + p.LastUpdated = c.Committer.When + } + + p.Created = c.Committer.When + } + + return p +} + +func main() { + posts, err := content.GetPostFiles(filepath.Join(builder.ContentRoot, "post")) + if err != nil { + log.Fatal("failed to glob posts", "error", err) + } + + var ( + wg sync.WaitGroup + mu sync.Mutex + ) + out := make(map[string]*content.PostVCSDates) + for _, f := range posts { + wg.Add(1) + go func(f string) { + repo, err := vcs.Open(".") + if err != nil { + log.Error("failed to open repository", "error", err) + } + + defer wg.Done() + iter, err := repo.FileLog(f) + if err != nil { + log.Error("failed to get commit log for file", "filename", f) + } + dates := getPostVCSDates(iter) + mu.Lock() + out[f] = dates + mu.Unlock() + }(f) + } + + wg.Wait() + j, err := json.MarshalIndent(out, "", " ") + if err != nil { + log.Fatal("failed to convert to JSON", "error", err) + } + + err = os.WriteFile(filepath.Join(builder.ContentRoot, content.DatesFilename), j, 0644) + if err != nil { + log.Fatal("could not write output file", "error", err) + } +} diff --git a/content/dates.json b/content/dates.json new file mode 100644 index 0000000..83f0b85 --- /dev/null +++ b/content/dates.json @@ -0,0 +1,54 @@ +{ + "content/post/a-new-site.md": { + "Created": "2014-06-08T11:31:54+01:00", + "LastUpdated": "2023-09-15T19:35:23+02:00" + }, + "content/post/back-again.md": { + "Created": "2017-05-06T18:02:45+02:00", + "LastUpdated": "2023-09-15T19:35:23+02:00" + }, + "content/post/cedit-and-paredit.md": { + "Created": "2014-08-09T10:17:48+01:00", + "LastUpdated": "2023-09-15T19:35:23+02:00" + }, + "content/post/emacs-package-archive-statistics.md": { + "Created": "2014-07-19T13:24:29+01:00", + "LastUpdated": "2023-09-15T19:35:23+02:00" + }, + "content/post/git-cloning-similar-repositories.md": { + "Created": "2014-06-22T09:46:16+01:00", + "LastUpdated": "2023-09-15T19:35:23+02:00" + }, + "content/post/homesteading.md": { + "Created": "2023-09-22T12:38:34+02:00", + "LastUpdated": "2023-11-29T10:29:49+01:00" + }, + "content/post/nixos-on-nanopi-r5s.md": { + "Created": "2023-07-30T10:38:40+02:00", + "LastUpdated": "2024-04-21T22:07:12+02:00" + }, + "content/post/now-on-three-continents.md": { + "Created": "2023-07-02T10:01:36+02:00", + "LastUpdated": "2023-09-22T08:38:00+02:00" + }, + "content/post/opening-projects-with-projectile.md": { + "Created": "2014-07-12T11:57:34+01:00", + "LastUpdated": "2023-09-15T19:35:23+02:00" + }, + "content/post/postfix-as-null-client-with-external-catchall.md": { + "Created": "2020-09-11T20:17:15+02:00", + "LastUpdated": "2023-09-15T19:35:23+02:00" + }, + "content/post/repository-management-with-ghq.md": { + "Created": "2017-05-07T00:10:06+02:00", + "LastUpdated": "2023-09-15T19:35:23+02:00" + }, + "content/post/self-hosted-git.md": { + "Created": "2017-06-04T13:50:36+02:00", + "LastUpdated": "2024-04-21T20:36:01+02:00" + }, + "content/post/when-tailscale-magicdns-isn't.md": { + "Created": "2024-06-25T09:03:58+02:00", + "LastUpdated": "2024-06-25T09:03:58+02:00" + } +} \ No newline at end of file diff --git a/internal/builder/builder.go b/internal/builder/builder.go index a4e44c6..928e4df 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -14,6 +14,7 @@ import ( "website/internal/content" "website/internal/log" "website/internal/sitemap" + "website/internal/vcs" "website/templates" "github.com/a-h/templ" @@ -22,15 +23,18 @@ import ( ) type IOConfig struct { - Source string `conf:"default:.,short:s,flag:src"` - Destination string `conf:"default:public,short:d,flag:dest"` - Development bool `conf:"default:false,flag:dev"` + Source string `conf:"default:.,short:s,flag:src"` + Destination string `conf:"default:public,short:d,flag:dest"` + Development bool `conf:"default:false,flag:dev"` + Repository *vcs.Repository `conf:"-"` } type Result struct { Hashes []string } +const ContentRoot = "content" + var compressFiles = false func mkdirp(dirs ...string) error { @@ -110,7 +114,11 @@ func build(ioConfig *IOConfig, config *config.Config) (*Result, error) { return nil, errors.WithMessage(err, "could not create post output directory") } log.Debug("reading posts") - posts, tags, err := content.ReadPosts(joinSource("content"), "post", outDir) + posts, tags, err := content.ReadPosts(&content.Config{ + Root: joinSource(ContentRoot), + InputDir: "post", + OutputDir: outDir, + }) if err != nil { return nil, err } @@ -219,7 +227,7 @@ func build(ioConfig *IOConfig, config *config.Config) (*Result, error) { r.Hashes = append(r.Hashes, h) log.Debug("rendering homepage") - _, text, err := content.GetPost(joinSource(filepath.Join("content", "index.md"))) + _, text, err := content.GetPost(joinSource(filepath.Join(ContentRoot, "index.md"))) if err != nil { return nil, err } diff --git a/internal/content/posts.go b/internal/content/posts.go index 5185d06..a4fb4b4 100644 --- a/internal/content/posts.go +++ b/internal/content/posts.go @@ -3,7 +3,6 @@ package content import ( "bytes" "os" - "path" "path/filepath" "slices" "strings" @@ -28,6 +27,11 @@ type PostMatter struct { } `toml:"taxonomies"` } +type PostVCSDates struct { + Created time.Time + LastUpdated time.Time +} + type Post struct { Input string Output string @@ -39,6 +43,8 @@ type Post struct { type Tags mapset.Set[string] +const DatesFilename = "dates.json" + var markdown = goldmark.New( goldmark.WithRendererOptions( htmlrenderer.WithUnsafe(), @@ -79,47 +85,60 @@ func RenderMarkdown(content []byte) (string, error) { return buf.String(), nil } -func ReadPosts(root string, inputDir string, outputDir string) ([]Post, Tags, error) { +type Config struct { + Root string + InputDir string + OutputDir string +} + +func GetPostFiles(root string) ([]string, error) { + return filepath.Glob(filepath.Join(root, "*.md")) +} + +func ReadPosts(config *Config) ([]Post, Tags, error) { tags := mapset.NewSet[string]() posts := []Post{} - subdir := filepath.Join(root, inputDir) - files, err := os.ReadDir(subdir) + subdir := filepath.Join(config.Root, config.InputDir) + files, err := GetPostFiles(subdir) if err != nil { return nil, nil, errors.WithMessagef(err, "could not read post directory %s", subdir) } - outputReplacer := strings.NewReplacer(root, outputDir, ".md", "/index.html") - urlReplacer := strings.NewReplacer(root, "", ".md", "/") - for _, f := range files { - pathFromRoot := filepath.Join(subdir, f.Name()) - if !f.IsDir() && path.Ext(pathFromRoot) == ".md" { - output := outputReplacer.Replace(pathFromRoot) - url := urlReplacer.Replace(pathFromRoot) - log.Debug("reading post", "post", pathFromRoot) - matter, content, err := GetPost(pathFromRoot) - if err != nil { - return nil, nil, err - } - - for _, tag := range matter.Taxonomies.Tags { - tags.Add(strings.ToLower(tag)) - } - - log.Debug("rendering markdown in post", "post", pathFromRoot) - html, err := RenderMarkdown(content) - if err != nil { - return nil, nil, err - } - post := Post{ - Input: pathFromRoot, - Output: output, - Basename: filepath.Base(url), - URL: url, - PostMatter: *matter, - Content: html, - } - - posts = append(posts, post) + outputReplacer := strings.NewReplacer(config.Root, config.OutputDir, ".md", "/index.html") + urlReplacer := strings.NewReplacer(config.Root, "", ".md", "/") + for _, pathFromRoot := range files { + output := outputReplacer.Replace(pathFromRoot) + url := urlReplacer.Replace(pathFromRoot) + log.Debug("reading post", "post", pathFromRoot) + matter, content, err := GetPost(pathFromRoot) + if err != nil { + return nil, nil, err } + // iter, err := repo.FileLog(pathFromRoot) + // if err != nil { + // return nil, nil, err + // } + // dates := getPostVCSDates(iter) + // log.Debug("post dates", "created", dates.Created, "updated", dates.LastUpdated) + + for _, tag := range matter.Taxonomies.Tags { + tags.Add(strings.ToLower(tag)) + } + + log.Debug("rendering markdown in post", "post", pathFromRoot) + html, err := RenderMarkdown(content) + if err != nil { + return nil, nil, err + } + post := Post{ + Input: pathFromRoot, + Output: output, + Basename: filepath.Base(url), + URL: url, + PostMatter: *matter, + Content: html, + } + + posts = append(posts, post) } slices.SortFunc(posts, func(a, b Post) int { return b.Date.Compare(a.Date) diff --git a/internal/server/server.go b/internal/server/server.go index c7b5659..e83985d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -109,7 +109,7 @@ func New(runtimeConfig *Config) (*Server, error) { if err != nil { return nil, err } - _, err = vcs.CloneOrUpdate(vcsConfig) + builderConfig.Repository, err = vcs.CloneOrUpdate(vcsConfig) if err != nil { return nil, err } @@ -123,6 +123,12 @@ func New(runtimeConfig *Config) (*Server, error) { publicDir := filepath.Join(runtimeConfig.Root, "public") builderConfig.Destination = publicDir runtimeConfig.Root = publicDir + } else { + var err error + builderConfig.Repository, err = vcs.Open(".") + if err != nil { + return nil, err + } } config, err := cfg.GetConfig(builderConfig.Source) diff --git a/internal/vcs/repository.go b/internal/vcs/repository.go index 625fbd2..66c8c74 100644 --- a/internal/vcs/repository.go +++ b/internal/vcs/repository.go @@ -6,6 +6,7 @@ import ( "website/internal/log" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" "github.com/pkg/errors" ) @@ -28,14 +29,12 @@ func CloneOrUpdate(cfg *Config) (*Repository, error) { if !errors.Is(err, git.ErrRepositoryAlreadyExists) { return nil, err } - gr, err = git.PlainOpen(cfg.LocalPath) + repo, err := Open(cfg.LocalPath) if err != nil { return nil, err } - repo := &Repository{ - repo: gr, - } - _, err := repo.Update(cfg) + + _, err = repo.Update(cfg) if err != nil { return nil, err } @@ -48,6 +47,16 @@ func CloneOrUpdate(cfg *Config) (*Repository, error) { }, nil } +func Open(path string) (*Repository, error) { + gr, err := git.PlainOpen(path) + if err != nil { + return nil, err + } + return &Repository{ + repo: gr, + }, nil +} + func (r *Repository) Update(cfg *Config) (bool, error) { log.Info("updating repository") @@ -101,3 +110,10 @@ func (r *Repository) Clean(wt *git.Worktree) error { return nil } + +func (r *Repository) FileLog(filename string) (object.CommitIter, error) { + return r.repo.Log(&git.LogOptions{ + Order: git.LogOrderCommitterTime, + FileName: &filename, + }) +} diff --git a/internal/website/filemap.go b/internal/website/filemap.go index 28dcd40..ad3f6ae 100644 --- a/internal/website/filemap.go +++ b/internal/website/filemap.go @@ -10,8 +10,6 @@ import ( "path/filepath" "strings" - "website/internal/log" - "github.com/pkg/errors" ) @@ -94,7 +92,7 @@ func registerContentFiles(root string) error { case ".br", ".gz": return nil } - log.Debug("registering file", "urlpath", urlPath) + // log.Debug("registering file", "urlpath", urlPath) return registerFile(urlPath, filePath) } -- cgit 1.4.1