add rate limit support & automatic retry on hit
This commit is contained in:
GitHub
commit
70a0132f22
4
.github/workflows/ci.yml
vendored
4
.github/workflows/ci.yml
vendored
@ -22,7 +22,7 @@ jobs:
|
||||
- name: Set up latest Go 1.x version
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: '1.21'
|
||||
go-version: '1.22'
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
go mod tidy
|
||||
@ -36,7 +36,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: ['1.13', '1.14', '1.15', '1.16', '1.17', '1.18', '1.19', '1.20', '1.21']
|
||||
go-version: ['1.22']
|
||||
steps:
|
||||
- name: Set up Go ${{ matrix.go-version }}
|
||||
uses: actions/setup-go@v2
|
||||
|
||||
16
go.mod
16
go.mod
@ -1,10 +1,20 @@
|
||||
module github.com/retailcrm/mg-transport-api-client-go
|
||||
|
||||
go 1.13
|
||||
go 1.22
|
||||
|
||||
require (
|
||||
github.com/google/go-querystring v1.0.0
|
||||
github.com/stretchr/testify v1.4.0
|
||||
github.com/maypok86/otter v1.0.0
|
||||
github.com/stretchr/testify v1.8.1
|
||||
gopkg.in/h2non/gock.v1 v1.1.2
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dolthub/maphash v0.1.0 // indirect
|
||||
github.com/dolthub/swiss v0.2.1 // indirect
|
||||
github.com/gammazero/deque v0.2.1 // indirect
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
25
go.sum
25
go.sum
@ -1,20 +1,33 @@
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ=
|
||||
github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4=
|
||||
github.com/dolthub/swiss v0.2.1 h1:gs2osYs5SJkAaH5/ggVJqXQxRXtWshF6uE0lgR/Y3Gw=
|
||||
github.com/dolthub/swiss v0.2.1/go.mod h1:8AhKZZ1HK7g18j7v7k6c5cYIGEZJcPn0ARsai8cUrh0=
|
||||
github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0=
|
||||
github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU=
|
||||
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
|
||||
github.com/maypok86/otter v1.0.0 h1:nP13eaFQrfRQHD1vxEgdlqR9gLHvfW2VcS0hFitglIY=
|
||||
github.com/maypok86/otter v1.0.0/go.mod h1:koSPT30yWtqMNrFohaywMlgSHCuUg6IVqeDerwIM/Mg=
|
||||
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
|
||||
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY=
|
||||
gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@ -6,8 +6,11 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const MaxRPS = 100
|
||||
|
||||
var prefix = "/api/transport/v1"
|
||||
|
||||
// GetRequest performs GET request to the provided route.
|
||||
@ -60,6 +63,22 @@ func makeRequest(reqType, url string, buf io.Reader, c *MgClient) ([]byte, int,
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-Transport-Token", c.Token)
|
||||
|
||||
defer c.mux.Unlock()
|
||||
c.mux.Lock()
|
||||
|
||||
attempt := 0
|
||||
tryAgain:
|
||||
sleepTime := time.Second - time.Since(c.lastTime)
|
||||
if sleepTime < 0 {
|
||||
c.lastTime = time.Now()
|
||||
c.rps = 0
|
||||
} else if c.rps == MaxRPS {
|
||||
time.Sleep(sleepTime)
|
||||
c.lastTime = time.Now()
|
||||
c.rps = 0
|
||||
}
|
||||
c.rps++
|
||||
|
||||
if c.Debug {
|
||||
if strings.Contains(url, "/files/upload") {
|
||||
c.writeLog("MG TRANSPORT API Request: %s %s %s [file data]", reqType, url, c.Token)
|
||||
@ -73,6 +92,12 @@ func makeRequest(reqType, url string, buf io.Reader, c *MgClient) ([]byte, int,
|
||||
return res, 0, NewCriticalHTTPError(err)
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusTooManyRequests && attempt < 3 {
|
||||
attempt++
|
||||
c.writeLog("MG TRANSPORT API Request rate limit hit on attempt %d, retrying", attempt)
|
||||
goto tryAgain
|
||||
}
|
||||
|
||||
if resp.StatusCode >= http.StatusInternalServerError {
|
||||
err = NewServerError(resp)
|
||||
return res, resp.StatusCode, err
|
||||
|
||||
45
v1/storage.go
Normal file
45
v1/storage.go
Normal file
@ -0,0 +1,45 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/maypok86/otter"
|
||||
)
|
||||
|
||||
const mgClientCacheTTL = time.Hour * 1
|
||||
|
||||
var ErrNegativeCapacity = errors.New("capacity cannot be less than 1")
|
||||
|
||||
type MGClientPool struct {
|
||||
cache *otter.CacheWithVariableTTL[string, *MgClient]
|
||||
}
|
||||
|
||||
// NewMGClientPool initializes the client cache.
|
||||
func NewMGClientPool(capacity int) (*MGClientPool, error) {
|
||||
if capacity <= 0 {
|
||||
return nil, ErrNegativeCapacity
|
||||
}
|
||||
|
||||
cache, _ := otter.MustBuilder[string, *MgClient](capacity).WithVariableTTL().Build()
|
||||
return &MGClientPool{cache: &cache}, nil
|
||||
}
|
||||
|
||||
func (m *MGClientPool) Get(token string, url string) *MgClient {
|
||||
if client, ok := m.cache.Get(token); ok {
|
||||
return client
|
||||
}
|
||||
|
||||
client := New(url, token)
|
||||
m.cache.Set(token, client, mgClientCacheTTL)
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
func (m *MGClientPool) Remove(token string) {
|
||||
m.cache.Delete(token)
|
||||
}
|
||||
|
||||
func (m *MGClientPool) Close() {
|
||||
m.cache.Close()
|
||||
}
|
||||
31
v1/storage_test.go
Normal file
31
v1/storage_test.go
Normal file
@ -0,0 +1,31 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
type StorageTest struct {
|
||||
suite.Suite
|
||||
}
|
||||
|
||||
func TestStorage(t *testing.T) {
|
||||
suite.Run(t, new(StorageTest))
|
||||
}
|
||||
|
||||
func (t *StorageTest) Test_MGClientPool() {
|
||||
clientPool, err := NewMGClientPool(1)
|
||||
t.Assert().NoError(err)
|
||||
|
||||
client := clientPool.Get("test_token", "test_url")
|
||||
t.Assert().Equal("test_url", client.URL)
|
||||
|
||||
clientPool.Remove("test_token")
|
||||
clientPool.Close()
|
||||
}
|
||||
|
||||
func (t *StorageTest) Test_NegativeCapacity() {
|
||||
_, err := NewMGClientPool(-1)
|
||||
t.Assert().Equal(ErrNegativeCapacity.Error(), err.Error())
|
||||
}
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@ -83,6 +84,9 @@ type MgClient struct {
|
||||
Debug bool `json:"debug"`
|
||||
httpClient *http.Client `json:"-"`
|
||||
logger BasicLogger `json:"-"`
|
||||
mux sync.Mutex `json:"-"`
|
||||
lastTime time.Time `json:"-"`
|
||||
rps int `json:"-"`
|
||||
}
|
||||
|
||||
// Channel type.
|
||||
|
||||
@ -306,22 +306,19 @@ func TestUnmarshalMessageWebhook(t *testing.T) {
|
||||
},
|
||||
"template": {
|
||||
"code": "f87e678f_660b_461a_b60a_a6194e2e9094#thanks_for_order#ru",
|
||||
"args": [
|
||||
"8061C",
|
||||
"17400"
|
||||
],
|
||||
"variables": {
|
||||
"body": [
|
||||
"8061C",
|
||||
"17400"
|
||||
],
|
||||
"buttons": [
|
||||
[],
|
||||
[
|
||||
"8061"
|
||||
]
|
||||
]
|
||||
}
|
||||
"variables": {
|
||||
"body": {
|
||||
"args": [
|
||||
"8061C",
|
||||
"17400"
|
||||
]
|
||||
},
|
||||
"buttons": [{
|
||||
"args": [
|
||||
"8061"
|
||||
]
|
||||
}]
|
||||
}
|
||||
},
|
||||
"attachments": {
|
||||
"suggestions": [
|
||||
|
||||
Loading…
Reference in New Issue
Block a user