From f1ce0412192175769dd2eb617c5b14707b76c1dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D1=83=D1=85=D0=B0=D0=BD=D0=BE=D0=B2=20=D0=94=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D0=BB=D0=B0?= Date: Mon, 12 Feb 2024 14:31:57 +0300 Subject: [PATCH 1/4] Handling limits and exceedings --- go.mod | 16 +++++++++++++--- go.sum | 25 +++++++++++++++++++------ v1/request.go | 24 ++++++++++++++++++++++++ v1/storage.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ v1/storage_test.go | 30 ++++++++++++++++++++++++++++++ v1/types.go | 4 ++++ 6 files changed, 135 insertions(+), 9 deletions(-) create mode 100644 v1/storage.go create mode 100644 v1/storage_test.go diff --git a/go.mod b/go.mod index db37fef..927ab2a 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,20 @@ module github.com/retailcrm/mg-transport-api-client-go -go 1.13 +go 1.22 require ( github.com/google/go-querystring v1.0.0 - github.com/stretchr/testify v1.4.0 + github.com/maypok86/otter v1.0.0 + github.com/stretchr/testify v1.8.1 gopkg.in/h2non/gock.v1 v1.1.2 - gopkg.in/yaml.v2 v2.4.0 // indirect +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dolthub/maphash v0.1.0 // indirect + github.com/dolthub/swiss v0.2.1 // indirect + github.com/gammazero/deque v0.2.1 // indirect + github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index cdd46b5..e5cbe5e 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,33 @@ -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ= +github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4= +github.com/dolthub/swiss v0.2.1 h1:gs2osYs5SJkAaH5/ggVJqXQxRXtWshF6uE0lgR/Y3Gw= +github.com/dolthub/swiss v0.2.1/go.mod h1:8AhKZZ1HK7g18j7v7k6c5cYIGEZJcPn0ARsai8cUrh0= +github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= +github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/maypok86/otter v1.0.0 h1:nP13eaFQrfRQHD1vxEgdlqR9gLHvfW2VcS0hFitglIY= +github.com/maypok86/otter v1.0.0/go.mod h1:koSPT30yWtqMNrFohaywMlgSHCuUg6IVqeDerwIM/Mg= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/v1/request.go b/v1/request.go index 64162a4..0bf02cf 100644 --- a/v1/request.go +++ b/v1/request.go @@ -6,8 +6,11 @@ import ( "io" "net/http" "strings" + "time" ) +const MaxRPS = 100 + var prefix = "/api/transport/v1" // GetRequest performs GET request to the provided route. @@ -68,11 +71,32 @@ func makeRequest(reqType, url string, buf io.Reader, c *MgClient) ([]byte, int, } } + c.mux.Lock() + defer c.mux.Unlock() + + attempt := 0 +tryAgain: + sleepTime := time.Second - time.Since(c.lastTime) + if sleepTime < 0 { + c.lastTime = time.Now() + c.rps = 0 + } else if c.rps == MaxRPS { + time.Sleep(sleepTime) + c.lastTime = time.Now() + c.rps = 0 + } + c.rps++ + resp, err := c.httpClient.Do(req) if err != nil { return res, 0, NewCriticalHTTPError(err) } + if resp.StatusCode == http.StatusTooManyRequests && attempt < 3 { + attempt++ + goto tryAgain + } + if resp.StatusCode >= http.StatusInternalServerError { err = NewServerError(resp) return res, resp.StatusCode, err diff --git a/v1/storage.go b/v1/storage.go new file mode 100644 index 0000000..eb98e61 --- /dev/null +++ b/v1/storage.go @@ -0,0 +1,45 @@ +package v1 + +import ( + "errors" + "time" + + "github.com/maypok86/otter" +) + +const mgClientCacheTTL = time.Hour * 1 + +var NegativeCapacity = errors.New("capacity cannot be less than 1") + +type MGClientPool struct { + cache *otter.CacheWithVariableTTL[string, *MgClient] +} + +// NewMGClientPool initializes the client cache +func NewMGClientPool(capacity int) (*MGClientPool, error) { + if capacity <= 0 { + return nil, NegativeCapacity + } + + cache, _ := otter.MustBuilder[string, *MgClient](capacity).WithVariableTTL().Build() + return &MGClientPool{cache: &cache}, nil +} + +func (m *MGClientPool) Get(token string, url string) *MgClient { + if client, ok := m.cache.Get(token); ok { + return client + } + + client := New(url, token) + m.cache.Set(token, client, mgClientCacheTTL) + + return client +} + +func (m *MGClientPool) Remove(token string) { + m.cache.Delete(token) +} + +func (m *MGClientPool) Close() { + m.cache.Close() +} diff --git a/v1/storage_test.go b/v1/storage_test.go new file mode 100644 index 0000000..0f57f70 --- /dev/null +++ b/v1/storage_test.go @@ -0,0 +1,30 @@ +package v1 + +import ( + "github.com/stretchr/testify/suite" + "testing" +) + +type StorageTest struct { + suite.Suite +} + +func TestStorage(t *testing.T) { + suite.Run(t, new(StorageTest)) +} + +func (t *StorageTest) Test_MGClientPool() { + clientPool, err := NewMGClientPool(1) + t.Assert().NoError(err) + + client := clientPool.Get("test_token", "test_url") + t.Assert().Equal("test_url", client.URL) + + clientPool.Remove("test_token") + clientPool.Close() +} + +func (t *StorageTest) Test_NegativeCapacity() { + _, err := NewMGClientPool(-1) + t.Assert().Equal(NegativeCapacity.Error(), err.Error()) +} diff --git a/v1/types.go b/v1/types.go index 01fdf1c..6caaabc 100644 --- a/v1/types.go +++ b/v1/types.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net/http" + "sync" "time" ) @@ -83,6 +84,9 @@ type MgClient struct { Debug bool `json:"debug"` httpClient *http.Client `json:"-"` logger BasicLogger `json:"-"` + mux sync.Mutex `json:"-"` + lastTime time.Time `json:"-"` + rps int `json:"-"` } // Channel type. From 0a0de1f66962f761b9fc1b16b7661c22f9210948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D1=83=D1=85=D0=B0=D0=BD=D0=BE=D0=B2=20=D0=94=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D0=BB=D0=B0?= Date: Mon, 12 Feb 2024 14:55:05 +0300 Subject: [PATCH 2/4] fix lint problem and test unmarshal message webhook --- v1/storage.go | 6 +++--- v1/storage_test.go | 5 +++-- v1/types_test.go | 29 +++++++++++++---------------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/v1/storage.go b/v1/storage.go index eb98e61..55495ab 100644 --- a/v1/storage.go +++ b/v1/storage.go @@ -9,16 +9,16 @@ import ( const mgClientCacheTTL = time.Hour * 1 -var NegativeCapacity = errors.New("capacity cannot be less than 1") +var ErrNegativeCapacity = errors.New("capacity cannot be less than 1") type MGClientPool struct { cache *otter.CacheWithVariableTTL[string, *MgClient] } -// NewMGClientPool initializes the client cache +// NewMGClientPool initializes the client cache. func NewMGClientPool(capacity int) (*MGClientPool, error) { if capacity <= 0 { - return nil, NegativeCapacity + return nil, ErrNegativeCapacity } cache, _ := otter.MustBuilder[string, *MgClient](capacity).WithVariableTTL().Build() diff --git a/v1/storage_test.go b/v1/storage_test.go index 0f57f70..ca41c28 100644 --- a/v1/storage_test.go +++ b/v1/storage_test.go @@ -1,8 +1,9 @@ package v1 import ( - "github.com/stretchr/testify/suite" "testing" + + "github.com/stretchr/testify/suite" ) type StorageTest struct { @@ -26,5 +27,5 @@ func (t *StorageTest) Test_MGClientPool() { func (t *StorageTest) Test_NegativeCapacity() { _, err := NewMGClientPool(-1) - t.Assert().Equal(NegativeCapacity.Error(), err.Error()) + t.Assert().Equal(ErrNegativeCapacity.Error(), err.Error()) } diff --git a/v1/types_test.go b/v1/types_test.go index b767b37..5e7899a 100644 --- a/v1/types_test.go +++ b/v1/types_test.go @@ -306,22 +306,19 @@ func TestUnmarshalMessageWebhook(t *testing.T) { }, "template": { "code": "f87e678f_660b_461a_b60a_a6194e2e9094#thanks_for_order#ru", - "args": [ - "8061C", - "17400" - ], - "variables": { - "body": [ - "8061C", - "17400" - ], - "buttons": [ - [], - [ - "8061" - ] - ] - } + "variables": { + "body": { + "args": [ + "8061C", + "17400" + ] + }, + "buttons": [{ + "args": [ + "8061" + ] + }] + } }, "attachments": { "suggestions": [ From 8ea0679a09270611d099e9eec53e700ad346be55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D1=83=D1=85=D0=B0=D0=BD=D0=BE=D0=B2=20=D0=94=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D0=BB=D0=B0?= Date: Mon, 12 Feb 2024 15:10:10 +0300 Subject: [PATCH 3/4] update go in ci --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2569ecd..bbc87eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: - name: Set up latest Go 1.x version uses: actions/setup-go@v2 with: - go-version: '1.21' + go-version: '1.22' - name: Get dependencies run: | go mod tidy @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go-version: ['1.13', '1.14', '1.15', '1.16', '1.17', '1.18', '1.19', '1.20', '1.21'] + go-version: ['1.22'] steps: - name: Set up Go ${{ matrix.go-version }} uses: actions/setup-go@v2 From 1012f97f08c3d79c67d8c7a30c3186b810b125a6 Mon Sep 17 00:00:00 2001 From: Neur0toxine Date: Tue, 13 Feb 2024 12:13:49 +0300 Subject: [PATCH 4/4] better logging for rate limiting --- v1/request.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/v1/request.go b/v1/request.go index 0bf02cf..74f2c7a 100644 --- a/v1/request.go +++ b/v1/request.go @@ -63,16 +63,8 @@ func makeRequest(reqType, url string, buf io.Reader, c *MgClient) ([]byte, int, req.Header.Set("Content-Type", "application/json") req.Header.Set("X-Transport-Token", c.Token) - if c.Debug { - if strings.Contains(url, "/files/upload") { - c.writeLog("MG TRANSPORT API Request: %s %s %s [file data]", reqType, url, c.Token) - } else { - c.writeLog("MG TRANSPORT API Request: %s %s %s %v", reqType, url, c.Token, buf) - } - } - - c.mux.Lock() defer c.mux.Unlock() + c.mux.Lock() attempt := 0 tryAgain: @@ -87,6 +79,14 @@ tryAgain: } c.rps++ + if c.Debug { + if strings.Contains(url, "/files/upload") { + c.writeLog("MG TRANSPORT API Request: %s %s %s [file data]", reqType, url, c.Token) + } else { + c.writeLog("MG TRANSPORT API Request: %s %s %s %v", reqType, url, c.Token, buf) + } + } + resp, err := c.httpClient.Do(req) if err != nil { return res, 0, NewCriticalHTTPError(err) @@ -94,6 +94,7 @@ tryAgain: if resp.StatusCode == http.StatusTooManyRequests && attempt < 3 { attempt++ + c.writeLog("MG TRANSPORT API Request rate limit hit on attempt %d, retrying", attempt) goto tryAgain }