feat: buffer fetched HTTP bodies

2 files changed, 25 insertions, 1 deletions

diff --git a/internal/fetcher/buf.go b/internal/fetcher/buf.go
new file mode 100644
index 0000000..6572130
--- /dev/null
+++ b/internal/fetcher/buf.go
@@ -0,0 +1,24 @@
+package fetcher
+
+import (
+	"bufio"
+	"io"
+
+	"gitlab.com/tozd/go/errors"
+)
+
+type Reader struct {
+	closer io.ReadCloser
+	*bufio.Reader
+}
+
+func NewReadCloser(body io.ReadCloser) *Reader {
+	return &Reader{
+		closer: body,
+		Reader: bufio.NewReader(body),
+	}
+}
+
+func (r *Reader) Close() error {
+	return errors.WithStack(r.closer.Close())
+}
diff --git a/internal/fetcher/http.go b/internal/fetcher/http.go
index ba99c3a..b0768ab 100644
--- a/internal/fetcher/http.go
+++ b/internal/fetcher/http.go
@@ -94,5 +94,5 @@ func fetchFileIfNeeded(
 		err = errors.Errorf("got response code %d, don't know what to do", res.StatusCode)
 	}
 
-	return body, newMtime, err
+	return NewReadCloser(body), newMtime, err
 }