2020-04-28 17:48:09 +03:00
|
|
|
package circuit
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2020-04-29 17:29:00 +03:00
|
|
|
"errors"
|
2020-04-28 17:48:09 +03:00
|
|
|
"time"
|
|
|
|
)
|
|
|
|
|
2020-04-29 17:29:00 +03:00
|
|
|
var (
|
|
|
|
ErrServiceUnavailable = errors.New("Service Unavailable")
|
|
|
|
)
|
|
|
|
|
2020-04-28 17:48:09 +03:00
|
|
|
type State int
|
|
|
|
|
|
|
|
const (
|
|
|
|
UnknownState State = iota
|
|
|
|
FailureState
|
|
|
|
SuccessState
|
|
|
|
)
|
|
|
|
|
2020-04-29 17:29:00 +03:00
|
|
|
//Counter interface
|
2020-04-28 17:48:09 +03:00
|
|
|
type Counter interface {
|
|
|
|
Count(State)
|
|
|
|
ConsecutiveFailures() uint32
|
|
|
|
LastActivity() time.Time
|
|
|
|
Reset()
|
|
|
|
}
|
|
|
|
|
2020-04-29 17:29:00 +03:00
|
|
|
type counters struct {
|
|
|
|
state State
|
|
|
|
lastActivity time.Time
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *counters) Count(State) {
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *counters) ConsecutiveFailures() uint32 {
|
|
|
|
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *counters) LastActivity() time.Time {
|
|
|
|
return c.lastActivity
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *counters) Reset() {
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewCounter() Counter {
|
|
|
|
var i Counter
|
|
|
|
return i
|
|
|
|
}
|
|
|
|
|
2020-04-28 17:48:09 +03:00
|
|
|
type Circuit func(context.Context) error
|
|
|
|
|
|
|
|
func Breaker(c Circuit, failureThreshold uint32) Circuit {
|
2020-04-29 17:29:00 +03:00
|
|
|
|
2020-04-28 17:48:09 +03:00
|
|
|
cnt := NewCounter()
|
|
|
|
|
2020-04-29 17:29:00 +03:00
|
|
|
return func(ctx context.Context) error {
|
2020-04-28 17:48:09 +03:00
|
|
|
if cnt.ConsecutiveFailures() >= failureThreshold {
|
2020-04-29 17:29:00 +03:00
|
|
|
canRetry := func(cnt Counter) bool {
|
|
|
|
backoffLevel := cnt.ConsecutiveFailures() - failureThreshold
|
2020-04-28 17:48:09 +03:00
|
|
|
|
|
|
|
// Calculates when should the circuit breaker resume propagating requests
|
|
|
|
// to the service
|
2020-04-29 17:29:00 +03:00
|
|
|
shouldRetryAt := cnt.LastActivity().Add(time.Second * 2 << backoffLevel)
|
2020-04-28 17:48:09 +03:00
|
|
|
|
|
|
|
return time.Now().After(shouldRetryAt)
|
|
|
|
}
|
|
|
|
|
|
|
|
if !canRetry(cnt) {
|
|
|
|
// Fails fast instead of propagating requests to the circuit since
|
|
|
|
// not enough time has passed since the last failure to retry
|
|
|
|
return ErrServiceUnavailable
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Unless the failure threshold is exceeded the wrapped service mimics the
|
|
|
|
// old behavior and the difference in behavior is seen after consecutive failures
|
|
|
|
if err := c(ctx); err != nil {
|
|
|
|
cnt.Count(FailureState)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
cnt.Count(SuccessState)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|