arian/navidrome
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Add a cached http client

Browse Source
This commit is contained in:
Deluan
2021-02-07 23:26:05 -05:00
committed by Deluan Quintão
parent 9d24106066
commit 28cdf1e693
11 changed files with 234 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/agents/agents_suite_test.go
+17
View File
@@ -0,0 +1,17 @@
package agents
import (
"testing"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/tests"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestAgents(t *testing.T) {
tests.Init(t, false)
log.SetLevel(log.LevelCritical)
RegisterFailHandler(Fail)
RunSpecs(t, "Agents Test Suite")
}
core/agents/cached_http_client.go
+102
View File
@@ -0,0 +1,102 @@
package agents
import (
"bufio"
"bytes"
"encoding/base64"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"strings"
"time"
"github.com/ReneKroon/ttlcache/v2"
"github.com/navidrome/navidrome/log"
)
const cacheSizeLimit = 1000
type CachedHTTPClient struct {
cache *ttlcache.Cache
hc httpDoer
}
type httpDoer interface {
Do(req *http.Request) (*http.Response, error)
}
type requestData struct {
Method string
Header http.Header
URL string
Body *string
}
func NewCachedHTTPClient(wrapped httpDoer, ttl time.Duration) *CachedHTTPClient {
c := &CachedHTTPClient{hc: wrapped}
c.cache = ttlcache.NewCache()
c.cache.SetCacheSizeLimit(cacheSizeLimit)
c.cache.SkipTTLExtensionOnHit(true)
c.cache.SetLoaderFunction(func(key string) (interface{}, time.Duration, error) {
req, err := c.deserializeReq(key)
if err != nil {
return nil, 0, err
}
resp, err := c.hc.Do(req)
if err != nil {
return nil, 0, err
}
return c.serializeResponse(resp), ttl, nil
})
c.cache.SetNewItemCallback(func(key string, value interface{}) {
log.Trace("New request cached", "req", key, "resp", value)
})
return c
}
func (c *CachedHTTPClient) Do(req *http.Request) (*http.Response, error) {
key := c.serializeReq(req)
respStr, err := c.cache.Get(key)
if err != nil {
return nil, err
}
return c.deserializeResponse(req, respStr.(string))
}
func (c *CachedHTTPClient) serializeReq(req *http.Request) string {
data := requestData{
Method: req.Method,
Header: req.Header,
URL: req.URL.String(),
}
if req.Body != nil {
bodyData, _ := ioutil.ReadAll(req.Body)
bodyStr := base64.StdEncoding.EncodeToString(bodyData)
data.Body = &bodyStr
}
j, _ := json.Marshal(&data)
return string(j)
}
func (c *CachedHTTPClient) deserializeReq(reqStr string) (*http.Request, error) {
var data requestData
_ = json.Unmarshal([]byte(reqStr), &data)
var body io.Reader
if data.Body != nil {
bodyStr, _ := base64.StdEncoding.DecodeString(*data.Body)
body = strings.NewReader(string(bodyStr))
}
return http.NewRequest(data.Method, data.URL, body)
}
func (c *CachedHTTPClient) serializeResponse(resp *http.Response) string {
var b = &bytes.Buffer{}
_ = resp.Write(b)
return b.String()
}
func (c *CachedHTTPClient) deserializeResponse(req *http.Request, respStr string) (*http.Response, error) {
r := bufio.NewReader(strings.NewReader(respStr))
return http.ReadResponse(r, req)
}
core/agents/cached_http_client_test.go
+81
View File
@@ -0,0 +1,81 @@
package agents
import (
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"time"
"github.com/navidrome/navidrome/consts"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("CachedHttpClient", func() {
Context("Default TTL", func() {
var chc *CachedHTTPClient
var ts *httptest.Server
var requestsReceived int
BeforeEach(func() {
ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestsReceived++
_, _ = fmt.Fprintf(w, "Hello, %s", r.URL.Query()["name"])
}))
chc = NewCachedHTTPClient(http.DefaultClient, consts.DefaultCachedHttpClientTTL)
})
AfterEach(func() {
defer ts.Close()
})
It("caches repeated requests", func() {
r, _ := http.NewRequest("GET", ts.URL+"?name=doe", nil)
resp, err := chc.Do(r)
Expect(err).To(BeNil())
body, err := ioutil.ReadAll(resp.Body)
Expect(err).To(BeNil())
Expect(string(body)).To(Equal("Hello, [doe]"))
Expect(requestsReceived).To(Equal(1))
// Same request
r, _ = http.NewRequest("GET", ts.URL+"?name=doe", nil)
resp, err = chc.Do(r)
Expect(err).To(BeNil())
body, err = ioutil.ReadAll(resp.Body)
Expect(err).To(BeNil())
Expect(string(body)).To(Equal("Hello, [doe]"))
Expect(requestsReceived).To(Equal(1))
// Different request
r, _ = http.NewRequest("GET", ts.URL, nil)
resp, err = chc.Do(r)
Expect(err).To(BeNil())
body, err = ioutil.ReadAll(resp.Body)
Expect(err).To(BeNil())
Expect(string(body)).To(Equal("Hello, []"))
Expect(requestsReceived).To(Equal(2))
})
It("expires responses after TTL", func() {
requestsReceived = 0
chc = NewCachedHTTPClient(http.DefaultClient, 10*time.Millisecond)
r, _ := http.NewRequest("GET", ts.URL+"?name=doe", nil)
_, err := chc.Do(r)
Expect(err).To(BeNil())
Expect(requestsReceived).To(Equal(1))
// Wait more than the TTL
time.Sleep(50 * time.Millisecond)
// Same request
r, _ = http.NewRequest("GET", ts.URL+"?name=doe", nil)
_, err = chc.Do(r)
Expect(err).To(BeNil())
Expect(requestsReceived).To(Equal(2))
})
})
})
core/agents/agents.go → core/agents/interfaces.go
View File
core/agents/lastfm.go
+6 -4
View File
@@ -5,6 +5,7 @@ import (
"net/http"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/core/lastfm"
"github.com/navidrome/navidrome/log"
)
@@ -26,7 +27,8 @@ func lastFMConstructor(ctx context.Context) Interface {
apiKey: conf.Server.LastFM.ApiKey,
lang: conf.Server.LastFM.Language,
}
l.client = lastfm.NewClient(l.apiKey, l.lang, http.DefaultClient)
hc := NewCachedHTTPClient(http.DefaultClient, consts.DefaultCachedHttpClientTTL)
l.client = lastfm.NewClient(l.apiKey, l.lang, hc)
return l
}
@@ -103,7 +105,7 @@ func (l *lastfmAgent) GetTopSongs(artistName, mbid string, count int) ([]Track,
func (l *lastfmAgent) callArtistGetInfo(name string, mbid string) (*lastfm.Artist, error) {
a, err := l.client.ArtistGetInfo(l.ctx, name)
if err != nil {
log.Error(l.ctx, "Error calling LastFM/artist.getInfo", "artist", name, "mbid", mbid)
log.Error(l.ctx, "Error calling LastFM/artist.getInfo", "artist", name, "mbid", mbid, err)
return nil, err
}
return a, nil
@@ -112,7 +114,7 @@ func (l *lastfmAgent) callArtistGetInfo(name string, mbid string) (*lastfm.Artis
func (l *lastfmAgent) callArtistGetSimilar(name string, mbid string, limit int) ([]lastfm.Artist, error) {
s, err := l.client.ArtistGetSimilar(l.ctx, name, limit)
if err != nil {
log.Error(l.ctx, "Error calling LastFM/artist.getSimilar", "artist", name, "mbid", mbid)
log.Error(l.ctx, "Error calling LastFM/artist.getSimilar", "artist", name, "mbid", mbid, err)
return nil, err
}
return s, nil
@@ -121,7 +123,7 @@ func (l *lastfmAgent) callArtistGetSimilar(name string, mbid string, limit int)
func (l *lastfmAgent) callArtistGetTopTracks(artistName, mbid string, count int) ([]lastfm.Track, error) {
t, err := l.client.ArtistGetTopTracks(l.ctx, artistName, count)
if err != nil {
log.Error(l.ctx, "Error calling LastFM/artist.getTopTracks", "artist", artistName, "mbid", mbid)
log.Error(l.ctx, "Error calling LastFM/artist.getTopTracks", "artist", artistName, "mbid", mbid, err)
return nil, err
}
return t, nil
core/agents/spotify.go
+3 -1
View File
@@ -8,6 +8,7 @@ import (
"strings"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/core/spotify"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
@@ -31,7 +32,8 @@ func spotifyConstructor(ctx context.Context) Interface {
id: conf.Server.Spotify.ID,
secret: conf.Server.Spotify.Secret,
}
l.client = spotify.NewClient(l.id, l.secret, http.DefaultClient)
hc := NewCachedHTTPClient(http.DefaultClient, consts.DefaultCachedHttpClientTTL)
l.client = spotify.NewClient(l.id, l.secret, hc)
return l
}
core/lastfm/client.go
+3 -3
View File
@@ -14,18 +14,18 @@ const (
apiBaseUrl = "https://ws.audioscrobbler.com/2.0/"
)
type HttpClient interface {
type httpDoer interface {
Do(req *http.Request) (*http.Response, error)
}
func NewClient(apiKey string, lang string, hc HttpClient) *Client {
func NewClient(apiKey string, lang string, hc httpDoer) *Client {
return &Client{apiKey, lang, hc}
}
type Client struct {
apiKey string
lang string
hc HttpClient
hc httpDoer
}
func (c *Client) makeRequest(params url.Values) (*Response, error) {
core/spotify/client.go
+6 -5
View File
@@ -21,18 +21,18 @@ var (
ErrNotFound = errors.New("spotify: not found")
)
type HttpClient interface {
type httpDoer interface {
Do(req *http.Request) (*http.Response, error)
}
func NewClient(id, secret string, hc HttpClient) *Client {
func NewClient(id, secret string, hc httpDoer) *Client {
return &Client{id, secret, hc}
}
type Client struct {
id string
secret string
hc HttpClient
hc httpDoer
}
func (c *Client) SearchArtists(ctx context.Context, name string, limit int) ([]Artist, error) {
@@ -66,9 +66,10 @@ func (c *Client) authorize(ctx context.Context) (string, error) {
payload := url.Values{}
payload.Add("grant_type", "client_credentials")
req, _ := http.NewRequest("POST", "https://accounts.spotify.com/api/token", strings.NewReader(payload.Encode()))
encodePayload := payload.Encode()
req, _ := http.NewRequest("POST", "https://accounts.spotify.com/api/token", strings.NewReader(encodePayload))
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
req.Header.Add("Content-Length", strconv.Itoa(len(payload.Encode())))
req.Header.Add("Content-Length", strconv.Itoa(len(encodePayload)))
auth := c.id + ":" + c.secret
req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(auth)))