diff options
29 files changed, 318 insertions, 169 deletions
diff --git a/cmd/searchix-web/main.go b/cmd/searchix-web/main.go index 500b6c7..63b1ec5 100644 --- a/cmd/searchix-web/main.go +++ b/cmd/searchix-web/main.go @@ -4,8 +4,6 @@ import ( "context" "flag" "fmt" - "log" - "log/slog" "os" "os/signal" @@ -13,6 +11,7 @@ import ( "go.alanpearce.eu/searchix" "go.alanpearce.eu/searchix/internal/config" + "go.alanpearce.eu/x/log" ) var ( @@ -22,10 +21,10 @@ var ( false, "print default configuration and exit", ) - liveReload = flag.Bool("live", false, "whether to enable live reloading (development)") - replace = flag.Bool("replace", false, "replace existing index and exit") - update = flag.Bool("update", false, "update index and exit") - version = flag.Bool("version", false, "print version information") + dev = flag.Bool("dev", false, "enable live reloading and nicer logging") + replace = flag.Bool("replace", false, "replace existing index and exit") + update = flag.Bool("update", false, "update index and exit") + version = flag.Bool("version", false, "print version information") ) func main() { @@ -45,23 +44,28 @@ func main() { os.Exit(0) } - cfg, err := config.GetConfig(*configFile) + logger := log.Configure(!*dev) + + cfg, err := config.GetConfig(*configFile, logger) if err != nil { - // only use log functions after the config file has been read successfully - log.Fatalf("Failed to parse config file: %v", err) + logger.Fatal("Failed to parse config file", "error", err) } - s, err := searchix.New(cfg) + + log.SetLevel(cfg.LogLevel) + + s, err := searchix.New(cfg, logger) if err != nil { - log.Fatalf("Failed to initialise searchix: %v", err) + logger.Fatal("Failed to initialise searchix", "error", err) } err = s.SetupIndex(&searchix.IndexOptions{ Update: *update, Replace: *replace, LowMemory: cfg.Importer.LowMemory, + Logger: logger, }) if err != nil { - log.Fatalf("Failed to setup index: %v", err) + logger.Fatal("Failed to setup index", "error", err) } if *replace || *update { @@ -72,15 +76,15 @@ func main() { defer cancel() go func() { - err = s.Start(ctx, *liveReload) + err = s.Start(ctx, *dev) if err != nil { // Error starting or closing listener: - log.Fatalf("error: %v", err) + logger.Fatal("error", "error", err) } }() <-ctx.Done() - slog.Debug("calling stop") + logger.Debug("calling stop") s.Stop() - slog.Debug("done") + logger.Debug("done") } diff --git a/defaults.toml b/defaults.toml index f35b539..bc1daf9 100644 --- a/defaults.toml +++ b/defaults.toml @@ -1,7 +1,7 @@ # Path to store index data. DataPath = './data' -# How much information to log, one of 'debug', 'info', 'warn', 'error'. -LogLevel = 'INFO' +# How much information to log, one of 'debug', 'info', 'warn', 'error', 'panic', 'fatal'. +LogLevel = 'info' # Settings for the web server [Web] diff --git a/go.mod b/go.mod index d27d308..ede6396 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module go.alanpearce.eu/searchix -go 1.22.2 +go 1.22.3 require ( badc0de.net/pkg/flagutil v1.0.1 @@ -16,13 +16,15 @@ require ( github.com/osdevisnot/sorvor v0.4.4 github.com/pelletier/go-toml/v2 v2.2.2 github.com/pkg/errors v0.9.1 - github.com/shengyanli1982/law v0.1.16 github.com/stoewer/go-strcase v1.3.0 github.com/yuin/goldmark v1.7.2 + go.alanpearce.eu/x v0.0.0-20240701200753-a70ddb349b02 + go.uber.org/zap v1.27.0 golang.org/x/net v0.26.0 ) require ( + github.com/Code-Hex/dd v1.1.0 // indirect github.com/RoaringBitmap/roaring v1.9.4 // indirect github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/blevesearch/geo v0.1.20 // indirect @@ -48,8 +50,12 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mschoch/smat v0.2.0 // indirect + github.com/sykesm/zap-logfmt v0.0.4 // indirect + github.com/thessem/zap-prettyconsole v0.5.0 // indirect go.etcd.io/bbolt v1.3.10 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect google.golang.org/protobuf v1.34.2 // indirect + moul.io/zapfilter v1.7.0 // indirect ) diff --git a/go.sum b/go.sum index 8cdbe2e..c373d7b 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,8 @@ badc0de.net/pkg/flagutil v1.0.1 h1:0ZgBzd3FehDUA8DJ70/phsnDH61/3aYMyx8Wd84KqQo= badc0de.net/pkg/flagutil v1.0.1/go.mod h1:HwwkfbImu+u288bnLaYDGqBxkJzvqi5YzKofmgkMLvk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Code-Hex/dd v1.1.0 h1:VEtTThnS9l7WhpKUIpdcWaf0B8Vp0LeeSEsxA1DZseI= +github.com/Code-Hex/dd v1.1.0/go.mod h1:VaMyo/YjTJ3d4qm/bgtrUkT2w+aYwJ07Y7eCWyrJr1w= github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/a-h/templ v0.2.707 h1:T1Gkd2ugbRglZ9rYw/VBchWOSZVKmetDbBkm4YubM7U= @@ -8,6 +11,7 @@ github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/bcicen/jstream v1.0.1 h1:BXY7Cu4rdmc0rhyTVyT3UkxAiX3bnLpKLas9btbH5ck= github.com/bcicen/jstream v1.0.1/go.mod h1:9ielPxqFry7Y4Tg3j4BfjPocfJ3TbsRtXOAYXYmRuAQ= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= @@ -68,8 +72,13 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -85,12 +94,14 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/shengyanli1982/law v0.1.16 h1:sQykz7ysBxYZSHkDdWj9C5EOE1Fez/PYg1bxij49Omg= -github.com/shengyanli1982/law v0.1.16/go.mod h1:20k9YnOTwilUB4X5Z4S7TIX5Ek1Ok4xfx8V8ZxIWlyM= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -98,6 +109,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -105,24 +117,94 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/sykesm/zap-logfmt v0.0.4 h1:U2WzRvmIWG1wDLCFY3sz8UeEmsdHQjHFNlIdmroVFaI= +github.com/sykesm/zap-logfmt v0.0.4/go.mod h1:AuBd9xQjAe3URrWT1BBDk2v2onAZHkZkWRMiYZXiZWA= +github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= +github.com/thessem/zap-prettyconsole v0.5.0 h1:AOu1GGUuDkGmj4tgRPSVf0vYGzDM+6cPWjKOcmjEcQs= +github.com/thessem/zap-prettyconsole v0.5.0/go.mod h1:3qfsE7y+bLOq7EQ+fMZHD3HYEp24ULFf5nhLSx6rjrE= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.7.2 h1:NjGd7lO7zrUn/A7eKwn5PEOt4ONYGqpxSEeZuduvgxc= github.com/yuin/goldmark v1.7.2/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= +go.alanpearce.eu/x v0.0.0-20240701200753-a70ddb349b02 h1:Ed0aWwSR9+Z7k/6LnG8iDXTW3Sb48Ahanjy7i83aboU= +go.alanpearce.eu/x v0.0.0-20240701200753-a70ddb349b02/go.mod h1:GaYgUfXSlaHBvdrInLYyKDMKo2Bmx1+IIFrlnZkZW+A= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.8.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +moul.io/zapfilter v1.7.0 h1:7aFrG4N72bDH9a2BtYUuUaDS981Dxu3qybWfeqaeBDU= +moul.io/zapfilter v1.7.0/go.mod h1:M+N2s+qZiA+bzRoyKMVRxyuERijS2ovi2pnMyiOGMvc= diff --git a/gomod2nix.toml b/gomod2nix.toml index e9650a5..bc25962 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -4,6 +4,9 @@ schema = 3 [mod."badc0de.net/pkg/flagutil"] version = "v1.0.1" hash = "sha256-0LRWL5DUHW3gXQhPAhUCxnUCN7HN1qKI2yZp8MrDN6M=" + [mod."github.com/Code-Hex/dd"] + version = "v1.1.0" + hash = "sha256-9aoekzjMXuJmR0/7bfu4y3SfcWBgdfYybB7gt4sNKfk=" [mod."github.com/RoaringBitmap/roaring"] version = "v1.9.4" hash = "sha256-OKOLQ/PsH6630Vb5/9yG28TLIPGxdG9WDbAZxgK8EcI=" @@ -115,18 +118,30 @@ schema = 3 [mod."github.com/pkg/errors"] version = "v0.9.1" hash = "sha256-mNfQtcrQmu3sNg/7IwiieKWOgFQOVVe2yXgKBpe/wZw=" - [mod."github.com/shengyanli1982/law"] - version = "v0.1.16" - hash = "sha256-UsO5qqKiREvwlz3JDKFAJFmXEu3JHYZOXibGgcgPNGY=" [mod."github.com/stoewer/go-strcase"] version = "v1.3.0" hash = "sha256-X0ilcefeqVQ44B9WT6euCMcigs7oLFypOQaGI33kGr8=" + [mod."github.com/sykesm/zap-logfmt"] + version = "v0.0.4" + hash = "sha256-KXVFtOU54chusK8AhZrzrvbbNmzq1mNrhs/7OmO+huE=" + [mod."github.com/thessem/zap-prettyconsole"] + version = "v0.5.0" + hash = "sha256-bOhManZjabZYHZwsaobaM9aPW+sUeqIfV+UnQLMaz54=" [mod."github.com/yuin/goldmark"] version = "v1.7.2" hash = "sha256-0rjUJP5WJy6227Epkgm/UHU9xzvrOAvYW+Y3EC+MkTE=" + [mod."go.alanpearce.eu/x"] + version = "v0.0.0-20240701200753-a70ddb349b02" + hash = "sha256-TRQgdPye/Q9LiM1XCDgxNrHTZKtSzuJ7lbNbWjkZvU4=" [mod."go.etcd.io/bbolt"] version = "v1.3.10" hash = "sha256-uEnz6jmmgT+hlwdZ8ns5NCJSbZcB4i123FF2cn2CbQA=" + [mod."go.uber.org/multierr"] + version = "v1.11.0" + hash = "sha256-Lb6rHHfR62Ozg2j2JZy3MKOMKdsfzd1IYTR57r3Mhp0=" + [mod."go.uber.org/zap"] + version = "v1.27.0" + hash = "sha256-8655KDrulc4Das3VRduO9MjCn8ZYD5WkULjCvruaYsU=" [mod."golang.org/x/net"] version = "v0.26.0" hash = "sha256-WfY33QERNbcIiDkH3+p2XGrAVqvWBQfc8neUt6TH6dQ=" @@ -139,3 +154,6 @@ schema = 3 [mod."google.golang.org/protobuf"] version = "v1.34.2" hash = "sha256-nMTlrDEE2dbpWz50eQMPBQXCyQh4IdjrTIccaU0F3m0=" + [mod."moul.io/zapfilter"] + version = "v1.7.0" + hash = "sha256-H6j5h8w123Y7d0zvKGkL5jiRYICtjmgzd2P/eeNaLrs=" diff --git a/internal/components/results.templ b/internal/components/results.templ index 4897638..b051219 100644 --- a/internal/components/results.templ +++ b/internal/components/results.templ @@ -2,16 +2,11 @@ package components import ( "strconv" - "log/slog" "go.alanpearce.eu/searchix/internal/nix" ) func convertMatch[I nix.Importable](m nix.Importable) *I { - i, ok := m.(I) - if !ok { - slog.Warn("Converting match failed", "match", m) - return nil - } + i := m.(I) return &i } diff --git a/internal/config/config.go b/internal/config/config.go index 14375d6..f3c0b57 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,7 +1,6 @@ package config import ( - "log/slog" "maps" "net/url" "os" @@ -9,6 +8,7 @@ import ( "github.com/pelletier/go-toml/v2" "github.com/pkg/errors" + "go.alanpearce.eu/x/log" ) var Version string @@ -102,10 +102,10 @@ func mustLocalTime(in string) (time LocalTime) { return } -func GetConfig(filename string) (*Config, error) { +func GetConfig(filename string, log *log.Logger) (*Config, error) { config := DefaultConfig if filename != "" { - slog.Debug("reading config", "filename", filename) + log.Debug("reading config", "filename", filename) f, err := os.Open(filename) if err != nil { return nil, errors.Wrap(err, "reading config failed") diff --git a/internal/config/structs.go b/internal/config/structs.go index e6cf20b..dd79303 100644 --- a/internal/config/structs.go +++ b/internal/config/structs.go @@ -5,14 +5,15 @@ package config import ( "fmt" - "log/slog" + + "go.uber.org/zap/zapcore" ) type Config struct { - DataPath string `comment:"Path to store index data."` - LogLevel slog.Level `comment:"How much information to log, one of 'debug', 'info', 'warn', 'error'."` - Web *Web `comment:"Settings for the web server"` - Importer *Importer `comment:"Settings for the import job"` + DataPath string `comment:"Path to store index data."` + LogLevel zapcore.Level `comment:"How much information to log, one of 'debug', 'info', 'warn', 'error', 'panic', 'fatal'."` + Web *Web `comment:"Settings for the web server"` + Importer *Importer `comment:"Settings for the import job"` } type Web struct { diff --git a/internal/fetcher/channel.go b/internal/fetcher/channel.go index 2bde631..8f0aa03 100644 --- a/internal/fetcher/channel.go +++ b/internal/fetcher/channel.go @@ -3,7 +3,6 @@ package fetcher import ( "context" "fmt" - "log/slog" "os" "os/exec" "path" @@ -13,6 +12,7 @@ import ( "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/index" + "go.alanpearce.eu/x/log" "github.com/pkg/errors" ) @@ -20,12 +20,12 @@ import ( type ChannelFetcher struct { Source *config.Source SourceFile string - Logger *slog.Logger + Logger *log.Logger } func NewChannelFetcher( source *config.Source, - logger *slog.Logger, + logger *log.Logger, ) (*ChannelFetcher, error) { switch source.Importer { case config.Options: diff --git a/internal/fetcher/download.go b/internal/fetcher/download.go index 941a683..a34c838 100644 --- a/internal/fetcher/download.go +++ b/internal/fetcher/download.go @@ -3,24 +3,24 @@ package fetcher import ( "context" "fmt" - "log/slog" "net/url" "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/index" "github.com/pkg/errors" + "go.alanpearce.eu/x/log" ) type DownloadFetcher struct { Source *config.Source SourceFile string - Logger *slog.Logger + Logger *log.Logger } func NewDownloadFetcher( source *config.Source, - logger *slog.Logger, + logger *log.Logger, ) (*DownloadFetcher, error) { switch source.Importer { case config.Options: @@ -59,7 +59,7 @@ func (i *DownloadFetcher) FetchIfNeeded( i.Logger.Debug("preparing to fetch URL", "url", fetchURL) - body, mtime, err := fetchFileIfNeeded(ctx, sourceUpdated, fetchURL) + body, mtime, err := fetchFileIfNeeded(ctx, i.Logger, sourceUpdated, fetchURL) if err != nil { i.Logger.Warn("failed to fetch file", "url", fetchURL, "error", err) diff --git a/internal/fetcher/http.go b/internal/fetcher/http.go index c848dc9..c5ec8fc 100644 --- a/internal/fetcher/http.go +++ b/internal/fetcher/http.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "log/slog" "net/http" "strings" "time" @@ -13,6 +12,7 @@ import ( "github.com/andybalholm/brotli" "github.com/pkg/errors" + "go.alanpearce.eu/x/log" ) type brotliReadCloser struct { @@ -33,6 +33,7 @@ func (r *brotliReadCloser) Close() error { func fetchFileIfNeeded( ctx context.Context, + log *log.Logger, mtime time.Time, url string, ) (body io.ReadCloser, newMtime time.Time, err error) { @@ -68,7 +69,7 @@ func fetchFileIfNeeded( case http.StatusOK: newMtime, err = time.Parse(time.RFC1123, res.Header.Get("Last-Modified")) if err != nil { - slog.Warn( + log.Warn( "could not parse Last-Modified header from response", "value", res.Header.Get("Last-Modified"), @@ -78,7 +79,7 @@ func fetchFileIfNeeded( switch ce := res.Header.Get("Content-Encoding"); ce { case "br": - slog.Debug("using brotli encoding") + log.Debug("using brotli encoding") body = newBrotliReader(res.Body) case "", "identity", "gzip": body = res.Body diff --git a/internal/fetcher/main.go b/internal/fetcher/main.go index fcc04a9..ac40ead 100644 --- a/internal/fetcher/main.go +++ b/internal/fetcher/main.go @@ -3,10 +3,10 @@ package fetcher import ( "context" "io" - "log/slog" "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/index" + "go.alanpearce.eu/x/log" "github.com/pkg/errors" ) @@ -23,7 +23,7 @@ type Fetcher interface { func New( source *config.Source, - logger *slog.Logger, + logger *log.Logger, ) (fetcher Fetcher, err error) { switch source.Fetcher { case config.ChannelNixpkgs: diff --git a/internal/fetcher/nixpkgs-channel.go b/internal/fetcher/nixpkgs-channel.go index ca33ae6..6f8ca63 100644 --- a/internal/fetcher/nixpkgs-channel.go +++ b/internal/fetcher/nixpkgs-channel.go @@ -3,18 +3,18 @@ package fetcher import ( "context" "fmt" - "log/slog" "net/url" "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/index" "github.com/pkg/errors" + "go.alanpearce.eu/x/log" ) type NixpkgsChannelFetcher struct { Source *config.Source - Logger *slog.Logger + Logger *log.Logger } func makeChannelURL(channel string, subPath string) (string, error) { @@ -25,7 +25,7 @@ func makeChannelURL(channel string, subPath string) (string, error) { func NewNixpkgsChannelFetcher( source *config.Source, - logger *slog.Logger, + logger *log.Logger, ) (*NixpkgsChannelFetcher, error) { switch source.Importer { case config.Options, config.Packages: @@ -66,7 +66,7 @@ func (i *NixpkgsChannelFetcher) FetchIfNeeded( } i.Logger.Debug("attempting to fetch file", "url", fetchURL) - body, mtime, err := fetchFileIfNeeded(ctx, sourceMeta.Updated, fetchURL) + body, mtime, err := fetchFileIfNeeded(ctx, i.Logger, sourceMeta.Updated, fetchURL) if err != nil { i.Logger.Warn("failed to fetch file", "url", fetchURL, "error", err) diff --git a/internal/importer/importer.go b/internal/importer/importer.go index 31d13c1..99f7e7a 100644 --- a/internal/importer/importer.go +++ b/internal/importer/importer.go @@ -2,11 +2,11 @@ package importer import ( "context" - "log/slog" "sync" "go.alanpearce.eu/searchix/internal/index" "go.alanpearce.eu/searchix/internal/nix" + "go.alanpearce.eu/x/log" ) type Importer interface { @@ -21,7 +21,7 @@ func process( ctx context.Context, indexer *index.WriteIndex, processor Processor, - logger *slog.Logger, + logger *log.Logger, ) (bool, error) { wg := sync.WaitGroup{} diff --git a/internal/importer/main.go b/internal/importer/main.go index bbd8b6c..4c66501 100644 --- a/internal/importer/main.go +++ b/internal/importer/main.go @@ -3,7 +3,6 @@ package importer import ( "context" "fmt" - "log/slog" "os/exec" "slices" "strings" @@ -12,18 +11,20 @@ import ( "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/fetcher" "go.alanpearce.eu/searchix/internal/index" + "go.alanpearce.eu/x/log" "github.com/pkg/errors" ) func createSourceImporter( parent context.Context, + log *log.Logger, meta *index.Meta, indexer *index.WriteIndex, forceUpdate bool, ) func(*config.Source) error { return func(source *config.Source) error { - logger := slog.With( + logger := log.With( "name", source.Key, "fetcher", @@ -94,9 +95,17 @@ func createSourceImporter( switch source.Importer { case config.Options: logger.Debug("processor created", "file", fmt.Sprintf("%T", files.Options)) - processor, err = NewOptionProcessor(files.Options, source) + processor, err = NewOptionProcessor( + files.Options, + source, + logger.Named("processor"), + ) case config.Packages: - processor, err = NewPackageProcessor(files.Packages, source) + processor, err = NewPackageProcessor( + files.Packages, + source, + logger.Named("processor"), + ) } if err != nil { return errors.WithMessagef(err, "failed to create processor") @@ -123,17 +132,18 @@ func createSourceImporter( func Start( cfg *config.Config, + log *log.Logger, indexer *index.WriteIndex, forceUpdate bool, onlyUpdateSources *[]string, ) error { if len(cfg.Importer.Sources) == 0 { - slog.Info("No sources enabled") + log.Info("No sources enabled") return nil } - slog.Debug("starting importer", "timeout", cfg.Importer.Timeout.Duration) + log.Debug("starting importer", "timeout", cfg.Importer.Timeout.Duration) importCtx, cancelImport := context.WithTimeout( context.Background(), cfg.Importer.Timeout.Duration, @@ -144,7 +154,7 @@ func Start( meta := indexer.Meta - importSource := createSourceImporter(importCtx, meta, indexer, forceUpdate) + importSource := createSourceImporter(importCtx, log, meta, indexer, forceUpdate) for name, source := range cfg.Importer.Sources { if onlyUpdateSources != nil && len(*onlyUpdateSources) > 0 { if !slices.Contains(*onlyUpdateSources, name) { @@ -153,7 +163,7 @@ func Start( } err := importSource(source) if err != nil { - slog.Error("import failed", "source", name, "error", err) + log.Error("import failed", "source", name, "error", err) } } diff --git a/internal/importer/main_test.go b/internal/importer/main_test.go index cb4d1db..0bc438d 100644 --- a/internal/importer/main_test.go +++ b/internal/importer/main_test.go @@ -1,26 +1,27 @@ package importer import ( - "log/slog" "testing" "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/index" + "go.alanpearce.eu/x/log" ) var cfg = config.DefaultConfig func BenchmarkImporterLowMemory(b *testing.B) { tmp := b.TempDir() - cfg.LogLevel = slog.LevelDebug + logger := log.Configure(false) _, write, _, err := index.OpenOrCreate(tmp, false, &index.Options{ LowMemory: true, + Logger: logger.Named("index"), }) if err != nil { b.Fatal(err) } - err = Start(&cfg, write, false, &[]string{"nixpkgs"}) + err = Start(&cfg, logger.Named("importer"), write, false, &[]string{"nixpkgs"}) if err != nil { b.Fatal(err) } diff --git a/internal/importer/options.go b/internal/importer/options.go index 290e2e3..763f57f 100644 --- a/internal/importer/options.go +++ b/internal/importer/options.go @@ -3,11 +3,11 @@ package importer import ( "context" "io" - "log/slog" "reflect" "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/nix" + "go.alanpearce.eu/x/log" "github.com/bcicen/jstream" "github.com/mitchellh/mapstructure" @@ -35,7 +35,7 @@ type nixOptionJSON struct { Type string } -func convertValue(nj *nixValueJSON) *nix.Value { +func (i *OptionIngester) convertValue(nj *nixValueJSON) *nix.Value { if nj == nil { return nil } @@ -49,7 +49,7 @@ func convertValue(nj *nixValueJSON) *nix.Value { Markdown: nix.Markdown(nj.Text), } default: - slog.Warn("got unexpected Value type", "type", nj.Type, "text", nj.Text) + i.log.Warn("got unexpected Value type", "type", nj.Type, "text", nj.Text) return nil } @@ -58,14 +58,20 @@ func convertValue(nj *nixValueJSON) *nix.Value { type OptionIngester struct { dec *jstream.Decoder ms *mapstructure.Decoder + log *log.Logger optJSON nixOptionJSON infile io.ReadCloser source *config.Source } -func NewOptionProcessor(infile io.ReadCloser, source *config.Source) (*OptionIngester, error) { +func NewOptionProcessor( + infile io.ReadCloser, + source *config.Source, + log *log.Logger, +) (*OptionIngester, error) { i := OptionIngester{ dec: jstream.NewDecoder(infile, 1).EmitKV(), + log: log, optJSON: nixOptionJSON{}, infile: infile, source: source, @@ -163,14 +169,14 @@ func (i *OptionIngester) Process(ctx context.Context) (<-chan nix.Importable, <- decs[i] = nix.Link(d) } - // slog.Debug("sending option", "name", kv.Key) + // log.Debug("sending option", "name", kv.Key) results <- nix.Option{ Name: kv.Key, Source: i.source.Key, Declarations: decs, - Default: convertValue(i.optJSON.Default), + Default: i.convertValue(i.optJSON.Default), Description: nix.Markdown(i.optJSON.Description), - Example: convertValue(i.optJSON.Example), + Example: i.convertValue(i.optJSON.Example), RelatedPackages: nix.Markdown(i.optJSON.RelatedPackages), Loc: i.optJSON.Loc, Type: i.optJSON.Type, diff --git a/internal/importer/package.go b/internal/importer/package.go index 5a0ea00..80adc38 100644 --- a/internal/importer/package.go +++ b/internal/importer/package.go @@ -9,6 +9,7 @@ import ( "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/nix" + "go.alanpearce.eu/x/log" "github.com/bcicen/jstream" "github.com/mitchellh/mapstructure" @@ -40,6 +41,7 @@ type maintainerJSON struct { type PackageIngester struct { dec *jstream.Decoder ms *mapstructure.Decoder + log *log.Logger pkg packageJSON infile io.ReadCloser source *config.Source @@ -60,9 +62,14 @@ func makeAdhocPlatform(v any) string { return string(s) } -func NewPackageProcessor(infile io.ReadCloser, source *config.Source) (*PackageIngester, error) { +func NewPackageProcessor( + infile io.ReadCloser, + source *config.Source, + log *log.Logger, +) (*PackageIngester, error) { i := &PackageIngester{ dec: jstream.NewDecoder(infile, 2).EmitKV(), + log: log, pkg: packageJSON{}, infile: infile, source: source, diff --git a/internal/index/index_meta.go b/internal/index/index_meta.go index fe1b26c..e67c6f2 100644 --- a/internal/index/index_meta.go +++ b/internal/index/index_meta.go @@ -2,11 +2,11 @@ package index import ( "encoding/json" - "log/slog" "os" "time" "go.alanpearce.eu/searchix/internal/file" + "go.alanpearce.eu/x/log" "github.com/pkg/errors" ) @@ -26,10 +26,11 @@ type data struct { type Meta struct { path string + log *log.Logger data } -func createMeta(path string) (*Meta, error) { +func createMeta(path string, log *log.Logger) (*Meta, error) { exists, err := file.Exists(path) if err != nil { return nil, errors.WithMessage(err, "could not check for existence of index metadata") @@ -40,19 +41,20 @@ func createMeta(path string) (*Meta, error) { return &Meta{ path: path, + log: log, data: data{ SchemaVersion: CurrentSchemaVersion, }, }, nil } -func openMeta(path string) (*Meta, error) { +func openMeta(path string, log *log.Logger) (*Meta, error) { exists, err := file.Exists(path) if err != nil { return nil, errors.WithMessage(err, "could not check for existence of index metadata") } if !exists { - return createMeta(path) + return createMeta(path, log) } j, err := os.ReadFile(path) @@ -61,6 +63,7 @@ func openMeta(path string) (*Meta, error) { } meta := Meta{ path: path, + log: log, } err = json.Unmarshal(j, &meta.data) if err != nil { @@ -74,7 +77,7 @@ func openMeta(path string) (*Meta, error) { func (i *Meta) checkSchemaVersion() { if i.SchemaVersion < CurrentSchemaVersion { - slog.Warn( + i.log.Warn( "Index schema version out of date, suggest re-indexing", "schema_version", i.SchemaVersion, @@ -90,7 +93,7 @@ func (i *Meta) Save() error { if err != nil { return errors.WithMessage(err, "could not prepare index metadata for saving") } - slog.Debug("saving index metadata", "path", i.path) + i.log.Debug("saving index metadata", "path", i.path) err = os.WriteFile(i.path, j, 0o600) if err != nil { return errors.WithMessage(err, "could not save index metadata") diff --git a/internal/index/indexer.go b/internal/index/indexer.go index 62edbc1..47701bd 100644 --- a/internal/index/indexer.go +++ b/internal/index/indexer.go @@ -5,8 +5,6 @@ import ( "context" "encoding/gob" "io/fs" - "log" - "log/slog" "math" "os" "path" @@ -14,6 +12,8 @@ import ( "go.alanpearce.eu/searchix/internal/file" "go.alanpearce.eu/searchix/internal/nix" + "go.alanpearce.eu/x/log" + "go.uber.org/zap" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" @@ -30,6 +30,7 @@ import ( type WriteIndex struct { index bleve.Index + log *log.Logger Meta *Meta } @@ -190,6 +191,7 @@ func deleteIndex(dataRoot string) error { type Options struct { LowMemory bool + Logger *log.Logger } func OpenOrCreate( @@ -198,7 +200,7 @@ func OpenOrCreate( options *Options, ) (*ReadIndex, *WriteIndex, bool, error) { var err error - bleve.SetLog(log.Default()) + bleve.SetLog(zap.NewStdLog(options.Logger.Named("bleve").GetLogger())) indexPath := path.Join(dataRoot, indexBaseName) metaPath := path.Join(dataRoot, metaBaseName) @@ -226,7 +228,7 @@ func OpenOrCreate( return nil, nil, false, err } - meta, err = createMeta(metaPath) + meta, err = createMeta(metaPath, options.Logger) if err != nil { return nil, nil, false, err } @@ -237,7 +239,7 @@ func OpenOrCreate( return nil, nil, exists, errors.WithMessagef(err, "could not open index at path %s", indexPath) } - meta, err = openMeta(metaPath) + meta, err = openMeta(metaPath, options.Logger) if err != nil { return nil, nil, exists, err } @@ -248,12 +250,14 @@ func OpenOrCreate( } return &ReadIndex{ - idx, - meta, + index: idx, + log: options.Logger, + meta: meta, }, &WriteIndex{ - idx, - meta, + index: idx, + log: options.Logger, + Meta: meta, }, exists, nil @@ -280,7 +284,7 @@ func (i *WriteIndex) Import( for obj := range objects { select { case <-ctx.Done(): - slog.Warn("import aborted") + i.log.Warn("import aborted") break outer default: @@ -305,7 +309,7 @@ func (i *WriteIndex) Import( field := document.NewTextFieldWithIndexingOptions("_data", nil, data.Bytes(), indexAPI.StoreField) newDoc := doc.AddField(field) - // slog.Debug("adding object to index", "name", opt.Name) + // log.Debug("adding object to index", "name", opt.Name) err = batch.IndexAdvanced(newDoc) if err != nil { @@ -340,7 +344,7 @@ func (i *WriteIndex) Flush(batch *bleve.Batch) error { error: errors.New("no documents to flush"), } } - slog.Debug("flushing batch", "size", size) + i.log.Debug("flushing batch", "size", size) err := i.index.Batch(batch) if err != nil { diff --git a/internal/index/search.go b/internal/index/search.go index dc19db4..03a8e60 100644 --- a/internal/index/search.go +++ b/internal/index/search.go @@ -7,6 +7,7 @@ import ( "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/nix" + "go.alanpearce.eu/x/log" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" @@ -29,6 +30,7 @@ type Result struct { type ReadIndex struct { index bleve.Index + log *log.Logger meta *Meta } diff --git a/internal/server/dev.go b/internal/server/dev.go index 17489d3..f5fd4fd 100644 --- a/internal/server/dev.go +++ b/internal/server/dev.go @@ -3,37 +3,41 @@ package server import ( "fmt" "io/fs" - "log/slog" "os" "path/filepath" "time" "github.com/fsnotify/fsnotify" "github.com/pkg/errors" + "go.alanpearce.eu/x/log" ) type FileWatcher struct { - *fsnotify.Watcher + watcher *fsnotify.Watcher + log *log.Logger } -func NewFileWatcher() (*FileWatcher, error) { +func NewFileWatcher(log *log.Logger) (*FileWatcher, error) { watcher, err := fsnotify.NewWatcher() if err != nil { return nil, errors.WithMessage(err, "could not create watcher") } - return &FileWatcher{watcher}, nil + return &FileWatcher{ + watcher, + log, + }, nil } -func (watcher FileWatcher) AddRecursive(from string) error { - slog.Debug(fmt.Sprintf("watching files under %s", from)) +func (i FileWatcher) AddRecursive(from string) error { + i.log.Debug(fmt.Sprintf("watching files under %s", from)) err := filepath.WalkDir(from, func(path string, entry fs.DirEntry, err error) error { if err != nil { return errors.WithMessagef(err, "could not walk directory %s", path) } if entry.IsDir() { - slog.Debug(fmt.Sprintf("adding directory %s to watcher", path)) - if err = watcher.Add(path); err != nil { + i.log.Debug(fmt.Sprintf("adding directory %s to watcher", path)) + if err = i.watcher.Add(path); err != nil { return errors.WithMessagef(err, "could not add directory %s to watcher", path) } } @@ -44,18 +48,18 @@ func (watcher FileWatcher) AddRecursive(from string) error { return errors.WithMessage(err, "error walking directory tree") } -func (watcher FileWatcher) Start(callback func(string)) { +func (i FileWatcher) Start(callback func(string)) { for { select { - case event := <-watcher.Events: + case event := <-i.watcher.Events: if event.Has(fsnotify.Create) || event.Has(fsnotify.Rename) { f, err := os.Stat(event.Name) if err != nil { - slog.Error(fmt.Sprintf("error handling %s event: %v", event.Op.String(), err)) + i.log.Error(fmt.Sprintf("error handling %s event: %v", event.Op.String(), err)) } else if f.IsDir() { - err = watcher.Add(event.Name) + err = i.watcher.Add(event.Name) if err != nil { - slog.Error(fmt.Sprintf("error adding new folder to watcher: %v", err)) + i.log.Error(fmt.Sprintf("error adding new folder to watcher: %v", err)) } } } @@ -63,8 +67,8 @@ func (watcher FileWatcher) Start(callback func(string)) { callback(event.Name) time.Sleep(500 * time.Millisecond) } - case err := <-watcher.Errors: - slog.Error(fmt.Sprintf("error in watcher: %v", err)) + case err := <-i.watcher.Errors: + i.log.Error(fmt.Sprintf("error in watcher: %v", err)) } } } diff --git a/internal/server/error.go b/internal/server/error.go index d9d6778..1e04bbc 100644 --- a/internal/server/error.go +++ b/internal/server/error.go @@ -1,15 +1,16 @@ package server import ( - "log/slog" "net/http" "go.alanpearce.eu/searchix/internal/components" "go.alanpearce.eu/searchix/internal/config" + "go.alanpearce.eu/x/log" ) func createErrorHandler( config *config.Config, + log *log.Logger, ) func(http.ResponseWriter, *http.Request, string, int) { return func(w http.ResponseWriter, r *http.Request, message string, code int) { var err error @@ -31,7 +32,7 @@ func createErrorHandler( err = components.ErrorPage(indexData).Render(r.Context(), w) } if err != nil { - slog.Error( + log.Error( "error rendering error page template", "error", err, diff --git a/internal/server/logging.go b/internal/server/logging.go index 372972f..6e2f7c8 100644 --- a/internal/server/logging.go +++ b/internal/server/logging.go @@ -1,11 +1,10 @@ package server import ( - "fmt" - "io" "net/http" "github.com/pkg/errors" + "go.alanpearce.eu/x/log" ) type LoggingResponseWriter struct { @@ -42,7 +41,7 @@ func NewLoggingResponseWriter(w http.ResponseWriter) *LoggingResponseWriter { type wrappedHandlerOptions struct { defaultHostname string - logger io.Writer + logger *log.Logger } func wrapHandlerWithLogging(wrappedHandler http.Handler, opts wrappedHandlerOptions) http.Handler { @@ -54,13 +53,17 @@ func wrapHandlerWithLogging(wrappedHandler http.Handler, opts wrappedHandlerOpti lw := NewLoggingResponseWriter(w) wrappedHandler.ServeHTTP(lw, r) statusCode := lw.statusCode - fmt.Fprintf( - opts.logger, - "%s %s %d %s %s\n", + opts.logger.Info( + "http request", + "scheme", scheme, + "method", r.Method, + "code", statusCode, + "path", r.URL.Path, + "location", lw.Header().Get("Location"), ) }) diff --git a/internal/server/mux.go b/internal/server/mux.go index 16a0150..17445e5 100644 --- a/internal/server/mux.go +++ b/internal/server/mux.go @@ -4,12 +4,9 @@ import ( "context" "encoding/xml" "fmt" - "io" - "log/slog" "math" "net/http" "net/url" - "os" "path" "strconv" "time" @@ -19,11 +16,11 @@ import ( "go.alanpearce.eu/searchix/internal/config" search "go.alanpearce.eu/searchix/internal/index" "go.alanpearce.eu/searchix/internal/opensearch" + "go.alanpearce.eu/x/log" sentryhttp "github.com/getsentry/sentry-go/http" "github.com/osdevisnot/sorvor/pkg/livereload" "github.com/pkg/errors" - "github.com/shengyanli1982/law" ) type HTTPError struct { @@ -57,6 +54,7 @@ func sortSources(ss map[string]*config.Source) { func NewMux( cfg *config.Config, index *search.ReadIndex, + log *log.Logger, liveReload bool, ) (*http.ServeMux, error) { if cfg == nil { @@ -70,7 +68,7 @@ func NewMux( }) sortSources(cfg.Importer.Sources) - errorHandler := createErrorHandler(cfg) + errorHandler := createErrorHandler(cfg, log) top := http.NewServeMux() mux := http.NewServeMux() @@ -118,7 +116,7 @@ func NewMux( return } - slog.Error("search error", "error", err) + log.Error("search error", "error", err) errorHandler(w, r, err.Error(), http.StatusInternalServerError) return @@ -177,7 +175,7 @@ func NewMux( err = components.ResultsPage(tdata).Render(r.Context(), w) } if err != nil { - slog.Error("template error", "template", importerType, "error", err) + log.Error("template error", "template", importerType, "error", err) errorHandler(w, r, err.Error(), http.StatusInternalServerError) } } else { @@ -258,7 +256,7 @@ func NewMux( err = components.DetailPage(tdata.TemplateData, *doc).Render(r.Context(), w) } if err != nil { - slog.Error("template error", "template", importerSingular, "error", err) + log.Error("template error", "template", importerSingular, "error", err) errorHandler(w, r, err.Error(), http.StatusInternalServerError) } } @@ -332,7 +330,7 @@ func NewMux( liveReload := livereload.New() liveReload.Start() top.Handle("/livereload", liveReload) - fw, err := NewFileWatcher() + fw, err := NewFileWatcher(log.Named("watcher")) if err != nil { return nil, errors.WithMessage(err, "could not create file watcher") } @@ -341,29 +339,23 @@ func NewMux( return nil, errors.WithMessage(err, "could not add directory to file watcher") } go fw.Start(func(filename string) { - slog.Debug(fmt.Sprintf("got filename %s", filename)) + log.Debug(fmt.Sprintf("got filename %s", filename)) if match, _ := path.Match("frontend/static/*", filename); match { err := frontend.Rehash() if err != nil { - slog.Error("failed to re-hash frontend assets", "error", err) + log.Error("failed to re-hash frontend assets", "error", err) } } liveReload.Reload() }) } - var logWriter io.Writer - if cfg.Web.Environment == "production" { - logWriter = law.NewWriteAsyncer(os.Stdout, nil) - } else { - logWriter = os.Stdout - } top.Handle("/", AddHeadersMiddleware( sentryHandler.Handle( wrapHandlerWithLogging(mux, wrappedHandlerOptions{ defaultHostname: cfg.Web.BaseURL.Hostname(), - logger: logWriter, + logger: log, }), ), cfg, diff --git a/internal/server/server.go b/internal/server/server.go index c3c2a4d..aacef30 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -2,7 +2,6 @@ package server import ( "context" - "log/slog" "net" "net/http" "strconv" @@ -10,6 +9,7 @@ import ( "go.alanpearce.eu/searchix/internal/config" "go.alanpearce.eu/searchix/internal/index" + "go.alanpearce.eu/x/log" "github.com/pkg/errors" "golang.org/x/net/http2" @@ -18,18 +18,25 @@ import ( type Server struct { cfg *config.Config + log *log.Logger server *http.Server listener net.Listener } -func New(conf *config.Config, index *index.ReadIndex, liveReload bool) (*Server, error) { - mux, err := NewMux(conf, index, liveReload) +func New( + conf *config.Config, + index *index.ReadIndex, + log *log.Logger, + liveReload bool, +) (*Server, error) { + mux, err := NewMux(conf, index, log, liveReload) if err != nil { return nil, err } return &Server{ cfg: conf, + log: log, server: &http.Server{ Handler: http.MaxBytesHandler( h2c.NewHandler(mux, &http2.Server{ @@ -56,7 +63,7 @@ func (s *Server) Start() error { s.listener = l if s.cfg.Web.Environment == "development" { - slog.Info( + s.log.Info( "server listening on", "url", s.cfg.Web.BaseURL.String(), @@ -71,19 +78,19 @@ func (s *Server) Start() error { } func (s *Server) Stop() chan struct{} { - slog.Debug("stop called") + s.log.Debug("stop called") idleConnsClosed := make(chan struct{}) go func() { - slog.Debug("shutting down server") + s.log.Debug("shutting down server") ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() err := s.server.Shutdown(ctx) - slog.Debug("server shut down") + s.log.Debug("server shut down") if err != nil { // Error from closing listeners, or context timeout: - slog.Error("error shutting down server", "error", err) + s.log.Error("error shutting down server", "error", err) } s.listener.Close() close(idleConnsClosed) diff --git a/justfile b/justfile index fbfbc2e..94b90ee 100644 --- a/justfile +++ b/justfile @@ -47,7 +47,7 @@ dev: modd reindex: - wgo run --exit ./cmd/searchix-web --config config.toml --replace + wgo run --exit ./cmd/searchix-web --config config.toml --replace --dev update: - wgo run --exit ./cmd/searchix-web --config config.toml --update + wgo run --exit ./cmd/searchix-web --config config.toml --update --dev diff --git a/modd.conf b/modd.conf index ab01682..df76d41 100644 --- a/modd.conf +++ b/modd.conf @@ -1,4 +1,4 @@ **/*.go !**/*_templ.go config.toml { daemon +sigint: templ generate --watch --proxy="http://localhost:3000" --open-browser=false \ - --cmd="go run ./cmd/searchix-web --live --config config.toml" + --cmd="go run ./cmd/searchix-web --dev --config config.toml" } diff --git a/searchix.go b/searchix.go index 1f14d95..558847f 100644 --- a/searchix.go +++ b/searchix.go @@ -2,8 +2,6 @@ package searchix import ( "context" - "log" - "log/slog" "slices" "sync" "time" @@ -12,6 +10,7 @@ import ( "go.alanpearce.eu/searchix/internal/importer" "go.alanpearce.eu/searchix/internal/index" "go.alanpearce.eu/searchix/internal/server" + "go.alanpearce.eu/x/log" "github.com/getsentry/sentry-go" "github.com/pelletier/go-toml/v2" @@ -42,6 +41,7 @@ type IndexOptions struct { Update bool Replace bool LowMemory bool + Logger *log.Logger } func (s *Server) SetupIndex(options *IndexOptions) error { @@ -58,6 +58,7 @@ func (s *Server) SetupIndex(options *IndexOptions) error { options.Replace, &index.Options{ LowMemory: options.LowMemory, + Logger: options.Logger.Named("index"), }, ) if err != nil { @@ -67,7 +68,7 @@ func (s *Server) SetupIndex(options *IndexOptions) error { s.writeIndex = write if !exists || options.Replace || options.Update { - slog.Info( + s.log.Info( "Starting build job", "new", !exists, @@ -76,7 +77,13 @@ func (s *Server) SetupIndex(options *IndexOptions) error { "update", options.Update, ) - err = importer.Start(s.cfg, write, options.Replace || options.Update, nil) + err = importer.Start( + s.cfg, + s.log.Named("importer"), + write, + options.Replace || options.Update, + nil, + ) if err != nil { return errors.Wrap(err, "Failed to build index") } @@ -97,14 +104,14 @@ func (s *Server) SetupIndex(options *IndexOptions) error { return slices.Contains(cfgEnabledSources, s) }) if len(newSources) > 0 { - slog.Info("adding new sources", "sources", newSources) - err := importer.Start(s.cfg, write, false, &newSources) + s.log.Info("adding new sources", "sources", newSources) + err := importer.Start(s.cfg, options.Logger.Named("importer"), write, false, &newSources) if err != nil { return errors.Wrap(err, "Failed to update index with new sources") } } if len(retiredSources) > 0 { - slog.Info("removing retired sources", "sources", retiredSources) + s.log.Info("removing retired sources", "sources", retiredSources) for _, s := range retiredSources { err := write.DeleteBySource(s) if err != nil { @@ -122,19 +129,13 @@ type Server struct { sv *server.Server wg *sync.WaitGroup cfg *config.Config + log *log.Logger sentryHub *sentry.Hub readIndex *index.ReadIndex writeIndex *index.WriteIndex } -func New(cfg *config.Config) (*Server, error) { - slog.SetLogLoggerLevel(cfg.LogLevel) - if cfg.Web.Environment == "production" { - log.SetFlags(0) - } else { - log.SetFlags(log.LstdFlags) - } - +func New(cfg *config.Config, log *log.Logger) (*Server, error) { err := sentry.Init(sentry.ClientOptions{ EnableTracing: true, TracesSampleRate: 0.01, @@ -142,11 +143,12 @@ func New(cfg *config.Config) (*Server, error) { Environment: cfg.Web.Environment, }) if err != nil { - slog.Warn("could not initialise sentry", "error", err) + log.Warn("could not initialise sentry", "error", err) } return &Server{ cfg: cfg, + log: log, sentryHub: sentry.CurrentHub(), }, nil } @@ -170,27 +172,27 @@ func (s *Server) startUpdateTimer( s.wg.Add(1) nextRun := nextOccurrenceOfLocalTime(s.cfg.Importer.UpdateAt.LocalTime) for { - slog.Debug("scheduling next run", "next-run", nextRun) + s.log.Debug("scheduling next run", "next-run", nextRun) select { case <-ctx.Done(): - slog.Debug("stopping scheduler") + s.log.Debug("stopping scheduler") s.wg.Done() return case <-time.After(time.Until(nextRun)): } s.wg.Add(1) - slog.Info("updating index") + s.log.Info("updating index") eventID := localHub.CaptureCheckIn(&sentry.CheckIn{ MonitorSlug: monitorSlug, Status: sentry.CheckInStatusInProgress, }, monitorConfig) - err = importer.Start(s.cfg, s.writeIndex, false, nil) + err = importer.Start(s.cfg, s.log.Named("importer"), s.writeIndex, false, nil) s.wg.Done() if err != nil { - slog.Warn("error updating index", "error", err) + s.log.Warn("error updating index", "error", err) localHub.CaptureException(err) localHub.CaptureCheckIn(&sentry.CheckIn{ @@ -199,7 +201,7 @@ func (s *Server) startUpdateTimer( Status: sentry.CheckInStatusError, }, monitorConfig) } else { - slog.Info("update complete") + s.log.Info("update complete") localHub.CaptureCheckIn(&sentry.CheckIn{ ID: *eventID, @@ -214,7 +216,7 @@ func (s *Server) startUpdateTimer( func (s *Server) Start(ctx context.Context, liveReload bool) error { var err error - s.sv, err = server.New(s.cfg, s.readIndex, liveReload) + s.sv, err = server.New(s.cfg, s.readIndex, s.log.Named("server"), liveReload) if err != nil { return errors.Wrap(err, "error setting up server") } |