arian/navidrome
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Don't cache transcoded files if the request was cancelled (#2041)

Browse Source 
* Don't cache transcoded files if the request was cancelled (or there was a transcoding error)

* Add context to logs

* Simplify Wait error handling

* Fix flaky test

* Change log level for "populating cache" error message

* Small cleanups
This commit is contained in:
Deluan Quintão
2022-12-18 12:22:12 -05:00
committed by GitHub
parent 54395e7e6a
commit 24d520882e
4 changed files with 157 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils/cache/file_caches.go
Vendored
+30 -13
View File
@@ -10,6 +10,7 @@ import (
"github.com/djherbis/fscache"
"github.com/dustin/go-humanize"
"github.com/hashicorp/go-multierror"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/log"
@@ -27,7 +28,7 @@ type FileCache interface {
Available(ctx context.Context) bool
}
func NewFileCache(name, cacheSize, cacheFolder string, maxItems int, getReader ReadFunc) *fileCache {
func NewFileCache(name, cacheSize, cacheFolder string, maxItems int, getReader ReadFunc) FileCache {
fc := &fileCache{
name: name,
cacheSize: cacheSize,
@@ -86,6 +87,16 @@ func (fc *fileCache) Available(ctx context.Context) bool {
return fc.ready && !fc.disabled
}
func (fc *fileCache) invalidate(ctx context.Context, key string) error {
if !fc.Available(ctx) {
return nil
}
if !fc.cache.Exists(key) {
return nil
}
return fc.cache.Remove(key)
}
func (fc *fileCache) Get(ctx context.Context, arg Item) (*CachedStream, error) {
if !fc.Available(ctx) {
reader, err := fc.getReader(ctx, arg)
@@ -109,10 +120,17 @@ func (fc *fileCache) Get(ctx context.Context, arg Item) (*CachedStream, error) {
if err != nil {
return nil, err
}
go copyAndClose(ctx, w, reader)
go func() {
if err := copyAndClose(w, reader); err != nil {
log.Debug(ctx, "Error populating cache", "key", key, err)
if err = fc.invalidate(ctx, key); err != nil {
log.Warn(ctx, "Error removing key from cache", "key", key, err)
}
}
}()
}
// If it is in the cache, check if the stream is done being written. If so, return a ReaderSeeker
// If it is in the cache, check if the stream is done being written. If so, return a ReadSeeker
if cached {
size := getFinalCachedSize(r)
if size >= 0 {
@@ -129,7 +147,7 @@ func (fc *fileCache) Get(ctx context.Context, arg Item) (*CachedStream, error) {
}
}
// All other cases, just return a Reader, without Seek capabilities
// All other cases, just return the cache reader, without Seek capabilities
return &CachedStream{Reader: r, Cached: cached}, nil
}
@@ -140,7 +158,6 @@ type CachedStream struct {
Cached bool
}
func (s *CachedStream) Seekable() bool { return s.Seeker != nil }
func (s *CachedStream) Close() error {
if s.Closer != nil {
return s.Closer.Close()
@@ -162,21 +179,21 @@ func getFinalCachedSize(r fscache.ReadAtCloser) int64 {
return -1
}
func copyAndClose(ctx context.Context, w io.WriteCloser, r io.Reader) {
func copyAndClose(w io.WriteCloser, r io.Reader) error {
_, err := io.Copy(w, r)
if err != nil {
log.Error(ctx, "Error copying data to cache", err)
err = fmt.Errorf("copying data to cache: %w", err)
}
if c, ok := r.(io.Closer); ok {
err = c.Close()
if err != nil {
log.Error(ctx, "Error closing source stream", err)
if cErr := c.Close(); cErr != nil {
err = multierror.Append(err, fmt.Errorf("closing source stream: %w", cErr))
}
}
err = w.Close()
if err != nil {
log.Error(ctx, "Error closing cache writer", err)
if cErr := w.Close(); cErr != nil {
err = multierror.Append(err, fmt.Errorf("closing cache writer: %w", cErr))
}
return err
}
func newFSCache(name, cacheSize, cacheFolder string, maxItems int) (fscache.Cache, error) {
utils/cache/file_caches_test.go
Vendored
+82 -10
View File
@@ -2,12 +2,14 @@ package cache
import (
"context"
"errors"
"io"
"os"
"path/filepath"
"strings"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
@@ -15,16 +17,18 @@ import (
// Call NewFileCache and wait for it to be ready
func callNewFileCache(name, cacheSize, cacheFolder string, maxItems int, getReader ReadFunc) *fileCache {
fc := NewFileCache(name, cacheSize, cacheFolder, maxItems, getReader)
Eventually(func() bool { return fc.Ready(context.TODO()) }).Should(BeTrue())
return fc
Eventually(func() bool { return fc.Ready(context.Background()) }).Should(BeTrue())
return fc.(*fileCache)
}
var _ = Describe("File Caches", func() {
BeforeEach(func() {
conf.Server.DataFolder, _ = os.MkdirTemp("", "file_caches")
})
AfterEach(func() {
_ = os.RemoveAll(conf.Server.DataFolder)
tmpDir, _ := os.MkdirTemp("", "file_caches")
DeferCleanup(func() {
configtest.SetupConfig()
_ = os.RemoveAll(tmpDir)
})
conf.Server.DataFolder = tmpDir
})
Describe("NewFileCache", func() {
@@ -56,7 +60,7 @@ var _ = Describe("File Caches", func() {
return strings.NewReader(arg.Key()), nil
})
// First call is a MISS
s, err := fc.Get(context.TODO(), &testArg{"test"})
s, err := fc.Get(context.Background(), &testArg{"test"})
Expect(err).To(BeNil())
Expect(s.Cached).To(BeFalse())
Expect(s.Closer).To(BeNil())
@@ -64,7 +68,7 @@ var _ = Describe("File Caches", func() {
// Second call is a HIT
called = false
s, err = fc.Get(context.TODO(), &testArg{"test"})
s, err = fc.Get(context.Background(), &testArg{"test"})
Expect(err).To(BeNil())
Expect(io.ReadAll(s)).To(Equal([]byte("test")))
Expect(s.Cached).To(BeTrue())
@@ -79,22 +83,90 @@ var _ = Describe("File Caches", func() {
return strings.NewReader(arg.Key()), nil
})
// First call is a MISS
s, err := fc.Get(context.TODO(), &testArg{"test"})
s, err := fc.Get(context.Background(), &testArg{"test"})
Expect(err).To(BeNil())
Expect(s.Cached).To(BeFalse())
Expect(io.ReadAll(s)).To(Equal([]byte("test")))
// Second call is also a MISS
called = false
s, err = fc.Get(context.TODO(), &testArg{"test"})
s, err = fc.Get(context.Background(), &testArg{"test"})
Expect(err).To(BeNil())
Expect(io.ReadAll(s)).To(Equal([]byte("test")))
Expect(s.Cached).To(BeFalse())
Expect(called).To(BeTrue())
})
Context("reader errors", func() {
When("creating a reader fails", func() {
It("does not cache", func() {
fc := callNewFileCache("test", "1KB", "test", 0, func(ctx context.Context, arg Item) (io.Reader, error) {
return nil, errors.New("failed")
})
_, err := fc.Get(context.Background(), &testArg{"test"})
Expect(err).To(MatchError("failed"))
})
})
When("reader returns error", func() {
It("does not cache", func() {
fc := callNewFileCache("test", "1KB", "test", 0, func(ctx context.Context, arg Item) (io.Reader, error) {
return errFakeReader{errors.New("read failure")}, nil
})
s, err := fc.Get(context.Background(), &testArg{"test"})
Expect(err).ToNot(HaveOccurred())
_, _ = io.Copy(io.Discard, s)
// TODO How to make the fscache reader return the underlying reader error?
//Expect(err).To(MatchError("read failure"))
// Data should not be cached (or eventually be removed from cache)
Eventually(func() bool {
s, _ = fc.Get(context.Background(), &testArg{"test"})
if s != nil {
return s.Cached
}
return false
}).Should(BeFalse())
})
})
When("context is canceled", func() {
It("does not cache", func() {
ctx, cancel := context.WithCancel(context.Background())
fc := callNewFileCache("test", "1KB", "test", 0, func(ctx context.Context, arg Item) (io.Reader, error) {
return &ctxFakeReader{ctx}, nil
})
s, err := fc.Get(ctx, &testArg{"test"})
Expect(err).ToNot(HaveOccurred())
cancel()
b := make([]byte, 10)
_, err = s.Read(b)
// TODO Should be context.Canceled error
Expect(err).To(MatchError(io.EOF))
// Data should not be cached (or eventually be removed from cache)
Eventually(func() bool {
s, _ = fc.Get(context.Background(), &testArg{"test"})
if s != nil {
return s.Cached
}
return false
}).Should(BeFalse())
})
})
})
})
})
type testArg struct{ s string }
func (t *testArg) Key() string { return t.s }
type errFakeReader struct{ err error }
func (e errFakeReader) Read([]byte) (int, error) { return 0, e.err }
type ctxFakeReader struct{ ctx context.Context }
func (e *ctxFakeReader) Read([]byte) (int, error) { return 0, e.ctx.Err() }