Merge pull request #6 from Neur0toxine/master

Different changes for load tests & overall correctness

README.md

@@ -1,6 +1,7 @@
 ## MG Transport Library
 [![Build Status](https://travis-ci.org/retailcrm/mg-transport-core.svg?branch=master)](https://travis-ci.org/retailcrm/mg-transport-core)
 [![codecov](https://codecov.io/gh/retailcrm/mg-transport-core/branch/master/graph/badge.svg)](https://codecov.io/gh/retailcrm/mg-transport-core)
+[![GoDoc](https://godoc.org/github.com/retailcrm/mg-transport-core/core?status.svg)](https://godoc.org/github.com/retailcrm/mg-transport-core/core)  
 This library provides different functions like error-reporting, logging, localization, etc. in order to make it easier to create transports.   
 Usage:
 ```go

cmd/transport-core-tool/main_test.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCoreTool_MigrationCommandExists(t *testing.T) {
+	found := false
+
+	for _, cmd := range parser.Commands() {
+		if cmd != nil && cmd.Name == "migration" {
+			found = true
+		}
+	}
+
+	assert.True(t, found)
+}

core/config.go

@@ -4,6 +4,7 @@ import (
 	"io/ioutil"
 	"path/filepath"
 	"regexp"
+	"time"
 
 	"github.com/op/go-logging"
 	"gopkg.in/yaml.v2"
@@ -28,6 +29,7 @@ type ConfigInterface interface {
 	GetDBConfig() DatabaseConfig
 	GetAWSConfig() ConfigAWS
 	GetTransportInfo() InfoInterface
+	GetHTTPClientConfig() *HTTPClientConfig
 	GetUpdateInterval() int
 	IsDebug() bool
 }
@@ -50,6 +52,7 @@ type Config struct {
 	UpdateInterval   int               `yaml:"update_interval"`
 	ConfigAWS        ConfigAWS         `yaml:"config_aws"`
 	TransportInfo    Info              `yaml:"transport_info"`
+	HTTPClientConfig *HTTPClientConfig `yaml:"http_client"`
 }
 
 // Info struct
@@ -79,6 +82,14 @@ type DatabaseConfig struct {
 	ConnectionLifetime int         `yaml:"connection_lifetime"`
 }
 
+// HTTPClientConfig struct
+type HTTPClientConfig struct {
+	Timeout         time.Duration `yaml:"timeout"`
+	SSLVerification bool          `yaml:"ssl_verification"`
+	MockAddress     string        `yaml:"mock_address"`
+	MockedDomains   []string      `yaml:"mocked_domains"`
+}
+
 // HTTPServerConfig struct
 type HTTPServerConfig struct {
 	Host   string `yaml:"host"`
@@ -168,6 +179,11 @@ func (c Config) GetUpdateInterval() int {
 	return c.UpdateInterval
 }
 
+// GetHTTPClientConfig returns http client config
+func (c Config) GetHTTPClientConfig() *HTTPClientConfig {
+	return c.HTTPClientConfig
+}
+
 // GetName transport name
 func (t Info) GetName() string {
 	return t.Name

core/config_test.go

@@ -20,7 +20,7 @@ type ConfigTest struct {
 	data   []byte
 }
 
-func (c *ConfigTest) SetupTest() {
+func (c *ConfigTest) SetupSuite() {
 	c.data = []byte(`
 version: 3.2.1
 
@@ -102,7 +102,7 @@ func (c *ConfigTest) Test_GetConfigAWS() {
 	assert.Equal(c.T(), "image/jpeg", c.config.GetAWSConfig().ContentType)
 }
 
-func (c *ConfigTest) TearDownTest() {
+func (c *ConfigTest) TearDownSuite() {
 	_ = os.Remove(testConfigFile)
 }
 

core/engine.go

@@ -2,6 +2,7 @@ package core
 
 import (
 	"html/template"
+	"net/http"
 
 	"github.com/gin-gonic/gin"
 	"github.com/gobuffalo/packr/v2"
@@ -15,6 +16,7 @@ type Engine struct {
 	Sentry
 	Utils
 	ginEngine    *gin.Engine
+	httpClient   *http.Client
 	Logger       *logging.Logger
 	Config       ConfigInterface
 	LogFormatter logging.Formatter
@@ -124,6 +126,37 @@ func (e *Engine) Router() *gin.Engine {
 	return e.ginEngine
 }
 
+// BuildHTTPClient builds HTTP client with provided configuration
+func (e *Engine) BuildHTTPClient(replaceDefault ...bool) *Engine {
+	if e.Config.GetHTTPClientConfig() != nil {
+		if client, err := NewHTTPClientBuilder().FromEngine(e).Build(replaceDefault...); err != nil {
+			panic(err)
+		} else {
+			e.httpClient = client
+		}
+	}
+
+	return e
+}
+
+// SetHTTPClient sets HTTP client to engine
+func (e *Engine) SetHTTPClient(client *http.Client) *Engine {
+	if client != nil {
+		e.httpClient = client
+	}
+
+	return e
+}
+
+// HTTPClient returns inner http client or default http client
+func (e *Engine) HTTPClient() *http.Client {
+	if e.httpClient == nil {
+		return http.DefaultClient
+	} else {
+		return e.httpClient
+	}
+}
+
 // ConfigureRouter will call provided callback with current gin.Engine, or panic if engine is not present
 func (e *Engine) ConfigureRouter(callback func(*gin.Engine)) *Engine {
 	callback(e.Router())

core/engine_test.go

@@ -147,6 +147,39 @@ func (e *EngineTest) Test_ConfigureRouter() {
 	})
 }
 
+func (e *EngineTest) Test_BuildHTTPClient() {
+	e.engine.Config = &Config{
+		HTTPClientConfig: &HTTPClientConfig{
+			Timeout:         30,
+			SSLVerification: true,
+		},
+	}
+	e.engine.BuildHTTPClient()
+
+	assert.NotNil(e.T(), e.engine.httpClient)
+}
+
+func (e *EngineTest) Test_SetHTTPClient() {
+	var err error
+
+	e.engine.httpClient = nil
+	e.engine.httpClient, err = NewHTTPClientBuilder().Build()
+
+	assert.NoError(e.T(), err)
+	assert.NotNil(e.T(), e.engine.httpClient)
+}
+
+func (e *EngineTest) Test_HTTPClient() {
+	var err error
+
+	e.engine.httpClient = nil
+	assert.NotNil(e.T(), e.engine.HTTPClient())
+
+	e.engine.httpClient, err = NewHTTPClientBuilder().Build()
+	assert.NoError(e.T(), err)
+	assert.NotNil(e.T(), e.engine.httpClient)
+}
+
 func (e *EngineTest) Test_Run_Fail() {
 	defer func() {
 		assert.NotNil(e.T(), recover())

core/http_client_builder.go

@@ -0,0 +1,252 @@
+package core
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	DefaultClient    = http.DefaultClient
+	DefaultTransport = http.DefaultTransport
+)
+
+// HTTPClientBuilder builds http client with mocks (if necessary) and timeout.
+// Example:
+// 		// Build HTTP client with timeout = 10 sec, without SSL certificates verification and with mocked google.com
+// 		client, err := NewHTTPClientBuilder().
+// 			SetTimeout(10).
+// 			SetMockAddress("api_mock:3004").
+// 			AddMockedDomain("google.com").
+// 			SetSSLVerification(false).
+// 			Build()
+//
+// 		if err != nil {
+// 			fmt.Print(err)
+// 		}
+//
+// 		// Actual response will be returned from "api_mock:3004" (it should provide any ssl certificate)
+// 		if resp, err := client.Get("https://google.com"); err == nil {
+// 			if data, err := ioutil.ReadAll(resp.Body); err == nil {
+// 				fmt.Printf("Data: %s", string(data))
+// 			} else {
+// 				fmt.Print(err)
+// 			}
+// 		} else {
+// 			fmt.Print(err)
+// 		}
+type HTTPClientBuilder struct {
+	httpClient    *http.Client
+	httpTransport *http.Transport
+	dialer        *net.Dialer
+	engine        *Engine
+	built         bool
+	logging       bool
+	timeout       time.Duration
+	mockAddress   string
+	mockHost      string
+	mockPort      string
+	mockedDomains []string
+}
+
+// NewHTTPClientBuilder returns HTTPClientBuilder with default values
+func NewHTTPClientBuilder() *HTTPClientBuilder {
+	return &HTTPClientBuilder{
+		built:         false,
+		httpClient:    &http.Client{},
+		httpTransport: &http.Transport{},
+		timeout:       30 * time.Second,
+		mockAddress:   "",
+		mockedDomains: []string{},
+		logging:       false,
+	}
+}
+
+// SetTimeout sets timeout for http client
+func (b *HTTPClientBuilder) SetTimeout(timeout time.Duration) *HTTPClientBuilder {
+	timeout = timeout * time.Second
+	b.timeout = timeout
+	b.httpClient.Timeout = timeout
+	return b
+}
+
+// SetMockAddress sets mock address
+func (b *HTTPClientBuilder) SetMockAddress(address string) *HTTPClientBuilder {
+	b.mockAddress = address
+	return b
+}
+
+// AddMockedDomain adds new mocked domain
+func (b *HTTPClientBuilder) AddMockedDomain(domain string) *HTTPClientBuilder {
+	b.mockedDomains = append(b.mockedDomains, domain)
+	return b
+}
+
+// SetMockedDomains sets mocked domains from slice
+func (b *HTTPClientBuilder) SetMockedDomains(domains []string) *HTTPClientBuilder {
+	b.mockedDomains = domains
+	return b
+}
+
+// SetSSLVerification enables or disables SSL certificates verification in client
+func (b *HTTPClientBuilder) SetSSLVerification(enabled bool) *HTTPClientBuilder {
+	if b.httpTransport.TLSClientConfig == nil {
+		b.httpTransport.TLSClientConfig = &tls.Config{}
+	}
+
+	b.httpTransport.TLSClientConfig.InsecureSkipVerify = !enabled
+
+	return b
+}
+
+// EnableLogging enables logging in mocks
+func (b *HTTPClientBuilder) EnableLogging() *HTTPClientBuilder {
+	b.logging = true
+	return b
+}
+
+// FromConfig fulfills mock configuration from HTTPClientConfig
+func (b *HTTPClientBuilder) FromConfig(config *HTTPClientConfig) *HTTPClientBuilder {
+	if config == nil {
+		return b
+	}
+
+	if config.MockAddress != "" {
+		b.SetMockAddress(config.MockAddress)
+		b.SetMockedDomains(config.MockedDomains)
+	}
+
+	if config.Timeout > 0 {
+		b.SetTimeout(config.Timeout)
+	}
+
+	b.SetSSLVerification(config.SSLVerification)
+
+	return b
+}
+
+// FromEngine fulfills mock configuration from ConfigInterface inside Engine
+func (b *HTTPClientBuilder) FromEngine(engine *Engine) *HTTPClientBuilder {
+	b.engine = engine
+	b.logging = engine.Config.IsDebug()
+	return b.FromConfig(engine.Config.GetHTTPClientConfig())
+}
+
+// buildDialer initializes dialer with provided timeout
+func (b *HTTPClientBuilder) buildDialer() *HTTPClientBuilder {
+	b.dialer = &net.Dialer{
+		Timeout:   b.timeout,
+		KeepAlive: b.timeout,
+	}
+
+	return b
+}
+
+// parseAddress parses address and returns error in case of error (port is necessary)
+func (b *HTTPClientBuilder) parseAddress() error {
+	if b.mockAddress == "" {
+		return nil
+	}
+
+	if host, port, err := net.SplitHostPort(b.mockAddress); err == nil {
+		b.mockHost = host
+		b.mockPort = port
+		return nil
+	} else {
+		return errors.Errorf("cannot split host and port: %s", err.Error())
+	}
+}
+
+// buildMocks builds mocks for http client
+func (b *HTTPClientBuilder) buildMocks() error {
+	if b.dialer == nil {
+		return errors.New("dialer must be built first")
+	}
+
+	if b.mockHost != "" && b.mockPort != "" && len(b.mockedDomains) > 0 {
+		b.logf("Mock address is \"%s\"\n", net.JoinHostPort(b.mockHost, b.mockPort))
+		b.logf("Mocked domains: ")
+
+		for _, domain := range b.mockedDomains {
+			b.logf(" - %s\n", domain)
+		}
+
+		b.httpTransport.DialContext = func(ctx context.Context, network, addr string) (conn net.Conn, e error) {
+			if host, port, err := net.SplitHostPort(addr); err != nil {
+				return b.dialer.DialContext(ctx, network, addr)
+			} else {
+				for _, mock := range b.mockedDomains {
+					if mock == host {
+						oldAddr := addr
+
+						if b.mockPort == "0" {
+							addr = net.JoinHostPort(b.mockHost, port)
+						} else {
+							addr = net.JoinHostPort(b.mockHost, b.mockPort)
+						}
+
+						b.logf("Mocking \"%s\" with \"%s\"\n", oldAddr, addr)
+					}
+				}
+			}
+
+			return b.dialer.DialContext(ctx, network, addr)
+		}
+	}
+
+	return nil
+}
+
+// logf prints logs via Engine or via fmt.Printf
+func (b *HTTPClientBuilder) logf(format string, args ...interface{}) {
+	if b.logging {
+		if b.engine != nil && b.engine.Logger != nil {
+			b.engine.Logger.Infof(format, args...)
+		} else {
+			fmt.Printf(format, args...)
+		}
+	}
+}
+
+// ReplaceDefault replaces default client and transport with generated ones
+func (b *HTTPClientBuilder) ReplaceDefault() *HTTPClientBuilder {
+	if b.built {
+		http.DefaultClient = b.httpClient
+		http.DefaultTransport = b.httpTransport
+	}
+
+	return b
+}
+
+// RestoreDefault restores default client and transport after replacement
+func (b *HTTPClientBuilder) RestoreDefault() *HTTPClientBuilder {
+	http.DefaultClient = DefaultClient
+	http.DefaultTransport = DefaultTransport
+
+	return b
+}
+
+// Build builds client, pass true to replace http.DefaultClient with generated one
+func (b *HTTPClientBuilder) Build(replaceDefault ...bool) (*http.Client, error) {
+	if err := b.buildDialer().parseAddress(); err != nil {
+		return nil, err
+	}
+
+	if err := b.buildMocks(); err != nil {
+		return nil, err
+	}
+
+	b.built = true
+	b.httpClient.Transport = b.httpTransport
+
+	if len(replaceDefault) > 0 && replaceDefault[0] {
+		b.ReplaceDefault()
+	}
+
+	return b.httpClient, nil
+}

core/http_client_builder_test.go

@@ -0,0 +1,135 @@
+package core
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+)
+
+type HTTPClientBuilderTest struct {
+	suite.Suite
+	builder *HTTPClientBuilder
+}
+
+func (t *HTTPClientBuilderTest) SetupSuite() {
+	t.builder = NewHTTPClientBuilder()
+}
+
+func (t *HTTPClientBuilderTest) Test_SetTimeout() {
+	t.builder.SetTimeout(90)
+
+	assert.Equal(t.T(), 90*time.Second, t.builder.timeout)
+	assert.Equal(t.T(), 90*time.Second, t.builder.httpClient.Timeout)
+}
+
+func (t *HTTPClientBuilderTest) Test_SetMockAddress() {
+	addr := "mock.local:3004"
+	t.builder.SetMockAddress(addr)
+
+	assert.Equal(t.T(), addr, t.builder.mockAddress)
+}
+
+func (t *HTTPClientBuilderTest) Test_AddMockedDomain() {
+	domain := "example.com"
+	t.builder.AddMockedDomain(domain)
+
+	assert.NotEmpty(t.T(), t.builder.mockedDomains)
+	assert.Equal(t.T(), domain, t.builder.mockedDomains[0])
+}
+
+func (t *HTTPClientBuilderTest) Test_SetMockedDomains() {
+	domains := []string{"example.com"}
+	t.builder.SetMockedDomains(domains)
+
+	assert.NotEmpty(t.T(), t.builder.mockedDomains)
+	assert.Equal(t.T(), domains[0], t.builder.mockedDomains[0])
+}
+
+func (t *HTTPClientBuilderTest) Test_SetSSLVerification() {
+	t.builder.SetSSLVerification(true)
+	assert.False(t.T(), t.builder.httpTransport.TLSClientConfig.InsecureSkipVerify)
+
+	t.builder.SetSSLVerification(false)
+	assert.True(t.T(), t.builder.httpTransport.TLSClientConfig.InsecureSkipVerify)
+}
+
+func (t *HTTPClientBuilderTest) Test_FromConfig() {
+	config := &HTTPClientConfig{
+		SSLVerification: true,
+		MockAddress:     "anothermock.local:3004",
+		MockedDomains:   []string{"example.gov"},
+		Timeout:         60,
+	}
+
+	t.builder.FromConfig(config)
+	assert.Equal(t.T(), !config.SSLVerification, t.builder.httpTransport.TLSClientConfig.InsecureSkipVerify)
+	assert.Equal(t.T(), config.MockAddress, t.builder.mockAddress)
+	assert.Equal(t.T(), config.MockedDomains[0], t.builder.mockedDomains[0])
+	assert.Equal(t.T(), config.Timeout*time.Second, t.builder.timeout)
+	assert.Equal(t.T(), config.Timeout*time.Second, t.builder.httpClient.Timeout)
+}
+
+func (t *HTTPClientBuilderTest) Test_FromEngine() {
+	engine := &Engine{
+		Config: Config{
+			HTTPClientConfig: &HTTPClientConfig{
+				SSLVerification: true,
+				MockAddress:     "anothermock.local:3004",
+				MockedDomains:   []string{"example.gov"},
+			},
+			Debug: false,
+		},
+	}
+
+	t.builder.FromEngine(engine)
+	assert.NotNil(t.T(), engine, t.builder.engine)
+}
+
+func (t *HTTPClientBuilderTest) Test_buildDialer() {
+	t.builder.buildDialer()
+
+	assert.NotNil(t.T(), t.builder.dialer)
+}
+
+func (t *HTTPClientBuilderTest) Test_parseAddress() {
+	assert.NoError(t.T(), t.builder.parseAddress())
+}
+
+func (t *HTTPClientBuilderTest) Test_buildMocks() {
+	assert.NoError(t.T(), t.builder.buildMocks())
+}
+
+func (t *HTTPClientBuilderTest) Test_logf() {
+	defer func() {
+		assert.Nil(t.T(), recover())
+	}()
+
+	t.builder.logf("test %s", "string")
+}
+
+func (t *HTTPClientBuilderTest) Test_Build() {
+	client, err := t.builder.
+		SetTimeout(10).
+		SetMockAddress("api_mock:3004").
+		AddMockedDomain("google.com").
+		Build(true)
+
+	assert.NoError(t.T(), err)
+	assert.NotNil(t.T(), client)
+	assert.Equal(t.T(), client, http.DefaultClient)
+}
+
+func (t *HTTPClientBuilderTest) Test_RestoreDefault() {
+	t.builder.ReplaceDefault()
+	t.builder.RestoreDefault()
+
+	assert.Equal(t.T(), http.DefaultClient, DefaultClient)
+	assert.Equal(t.T(), http.DefaultTransport, DefaultTransport)
+}
+
+func Test_HTTPClientBuilder(t *testing.T) {
+	suite.Run(t, new(HTTPClientBuilderTest))
+}

core/localizer_test.go

@@ -23,7 +23,7 @@ type LocalizerTest struct {
 	localizer *Localizer
 }
 
-func (l *LocalizerTest) SetupTest() {
+func (l *LocalizerTest) SetupSuite() {
 	if _, err := os.Stat(testTranslationsDir); err != nil && os.IsNotExist(err) {
 		err := os.Mkdir(testTranslationsDir, os.ModePerm)
 		require.Nil(l.T(), err)
@@ -78,7 +78,7 @@ func (l *LocalizerTest) Test_BadRequestLocalized() {
 	assert.Equal(l.T(), "Test message", resp.(ErrorResponse).Error)
 }
 
-func (l *LocalizerTest) TearDownTest() {
+func (l *LocalizerTest) TearDownSuite() {
 	err := os.RemoveAll(testTranslationsDir)
 	require.Nil(l.T(), err)
 }

core/migration_generator_test.go

@@ -0,0 +1,56 @@
+package core
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"os"
+	"path"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+)
+
+type MigrationGeneratorSuite struct {
+	suite.Suite
+	command *NewMigrationCommand
+}
+
+func (s *MigrationGeneratorSuite) SetupSuite() {
+	s.command = &NewMigrationCommand{Directory: "/tmp"}
+}
+
+func (s *MigrationGeneratorSuite) Test_FileExists() {
+	var (
+		seededRand *rand.Rand = rand.New(rand.NewSource(time.Now().UnixNano()))
+		notExist              = fmt.Sprintf("/tmp/%d", seededRand.Int31())
+	)
+
+	assert.False(s.T(), s.command.FileExists(notExist))
+}
+
+func (s *MigrationGeneratorSuite) Test_Execute() {
+	found := false
+	assert.NoError(s.T(), s.command.Execute([]string{}))
+	files, err := ioutil.ReadDir(s.command.Directory)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, f := range files {
+		if strings.Index(f.Name(), "_app.go") != -1 {
+			found = true
+			assert.NoError(s.T(), os.Remove(path.Join(s.command.Directory, f.Name())))
+		}
+	}
+
+	assert.True(s.T(), found)
+}
+
+func Test_MigrationGenerator(t *testing.T) {
+	suite.Run(t, new(MigrationGeneratorSuite))
+}

core/sentry_test.go

@@ -24,7 +24,7 @@ type SentryTest struct {
 	scalarTags *SentryTaggedScalar
 }
 
-func (s *SentryTest) SetupTest() {
+func (s *SentryTest) SetupSuite() {
 	s.structTags = NewTaggedStruct(SampleStruct{}, "struct", map[string]string{"fake": "prop"})
 	s.scalarTags = NewTaggedScalar("", "scalar", "Scalar")
 	require.Equal(s.T(), "struct", s.structTags.GetContextKey())

core/template_test.go

@@ -23,7 +23,7 @@ type TemplateTest struct {
 	renderer Renderer
 }
 
-func (t *TemplateTest) SetupTest() {
+func (t *TemplateTest) SetupSuite() {
 	if _, err := os.Stat(testTemplatesDir); err != nil && os.IsNotExist(err) {
 		err := os.Mkdir(testTemplatesDir, os.ModePerm)
 		require.Nil(t.T(), err)

core/utils_test.go

@@ -29,7 +29,7 @@ func mgClient() *v1.MgClient {
 	return v1.New(testMGURL, "token")
 }
 
-func (u *UtilsTest) SetupTest() {
+func (u *UtilsTest) SetupSuite() {
 	logger := NewLogger("code", logging.DEBUG, DefaultLogFormatter())
 	awsConfig := ConfigAWS{
 		AccessKeyID:     "access key id (will be removed)",
@@ -46,7 +46,7 @@ func (u *UtilsTest) SetupTest() {
 
 func (u *UtilsTest) Test_ResetUtils() {
 	assert.Equal(u.T(), "access key id (will be removed)", u.utils.ConfigAWS.AccessKeyID)
-	assert.Equal(u.T(), uint32(12345), u.utils.TokenCounter)
+	assert.Equal(u.T(), uint32(12346), u.utils.TokenCounter)
 	assert.False(u.T(), u.utils.IsDebug)
 
 	awsConfig := u.utils.ConfigAWS