add codes for circuit breaker

This commit is contained in:
Edward 2020-05-21 16:02:28 +08:00
parent 7bc707aeb3
commit 6ec8f1ebb5
4 changed files with 150 additions and 80 deletions

View File

@ -2,13 +2,22 @@ package circuit
import "time" import "time"
//BreakConditionWatcher check state
type BreakConditionWatcher func(counts counters) bool
//StateChangedEventHandler set event handle
type StateChangedEventHandler func(name string, from State, to State)
//Option set Options
type Option func(opts *Options)
//Options for breaker //Options for breaker
type Options struct { type Options struct {
Name string Name string
Expiry time.Time Expiry time.Time
Interval, Timeout time.Duration Interval, Timeout time.Duration
MaxRequests uint32 MaxRequests uint32
ReadyToTrip StateCheckerHandler WhenToBreak BreakConditionWatcher //是否应该断开电路(打开电路开关)
OnStateChanged StateChangedEventHandler OnStateChanged StateChangedEventHandler
} }
@ -58,9 +67,9 @@ func OnStateChanged(handler StateChangedEventHandler) Option {
} }
} }
//ReadyToTrip check traffic state ,to see if request can go //BreakIf check traffic state ,to see if request can go
func ReadyToTrip(readyToGo StateCheckerHandler) Option { func BreakIf(whenCondition BreakConditionWatcher) Option {
return func(opts *Options) { return func(opts *Options) {
opts.ReadyToTrip = readyToGo opts.WhenToBreak = whenCondition
} }
} }

View File

@ -5,12 +5,13 @@ package circuit
* @Author: Edward * @Author: Edward
* @Date: 2020-05-10 22:00:58 * @Date: 2020-05-10 22:00:58
* @Last Modified by: Edward * @Last Modified by: Edward
* @Last Modified time: 2020-05-11 22:15:25 * @Last Modified time: 2020-05-21 15:59:40
*/ */
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"sync" "sync"
"time" "time"
) )
@ -24,61 +25,14 @@ import (
var ( var (
ErrTooManyRequests = errors.New("too many requests") ErrTooManyRequests = errors.New("too many requests")
ErrServiceUnavailable = errors.New("service unavailable") ErrServiceUnavailable = errors.New("service unavailable")
FailureThreshold = 10 FailureThreshold = 10 //最大失败次数--->失败阈值
) )
//StateCheckerHandler check state ////////////////////////////////
type StateCheckerHandler func(counts counters) bool /// 状态计数器 用以维护断路器内部的状态
/// 无论是对象式断路器还是函数式断路器
//StateChangedEventHandler set event handle /// 都要用到计数器
type StateChangedEventHandler func(name string, from State, to State) ////////////////////////////////
//Option set Options
type Option func(opts *Options)
//RequestBreaker for protection
type RequestBreaker struct {
options Options
mutex sync.Mutex
state State
generation uint64
counts ICounter
}
//NewRequestBreaker return a breaker
func NewRequestBreaker(opts ...Option) *RequestBreaker {
defaultOptions := Options{
Name: "defaultBreakerName",
Expiry: time.Now().Add(time.Second * 20),
Interval: time.Second * 2,
Timeout: time.Second * 60, //default to 60 seconds
MaxRequests: 5,
ReadyToTrip: func(counts counters) bool { return true },
OnStateChanged: func(name string, from State, to State) {},
}
for _, setOption := range opts {
setOption(&defaultOptions)
}
return &RequestBreaker{
options: defaultOptions,
counts: nil,
generation: 0,
}
}
// Do the given requested work if the RequestBreaker accepts it.
// Do returns an error instantly if the RequestBreaker rejects the request.
// Otherwise, Execute returns the result of the request.
// If a panic occurs in the request, the RequestBreaker handles it as an error and causes the same panic again.
func (rb *RequestBreaker) Do(work func() (interface{}, error)) (interface{}, error) {
//do work from requested user
result, err := work()
return result, err
}
//State of current switch //State of current switch
type State int type State int
@ -90,18 +44,16 @@ const (
SuccessState SuccessState
) )
//Circuit of action stream
type Circuit func(context.Context) error
//ICounter interface //ICounter interface
type ICounter interface { type ICounter interface {
Count(State) Count(State)
LastActivity() time.Time LastActivity() time.Time
Reset() Reset()
Total() uint32
} }
type counters struct { type counters struct {
Requests uint32 Requests uint32 //连续的请求次数
lastState State lastState State
lastActivity time.Time lastActivity time.Time
counts uint32 //counts of failures counts uint32 //counts of failures
@ -111,12 +63,19 @@ type counters struct {
ConsecutiveFailures uint32 ConsecutiveFailures uint32
} }
func (c *counters) Total() uint32 {
return c.Requests
}
func (c *counters) LastActivity() time.Time { func (c *counters) LastActivity() time.Time {
return c.lastActivity return c.lastActivity
} }
func (c *counters) Reset() { func (c *counters) Reset() {
ct := &counters{}
ct.lastActivity = c.lastActivity
ct.lastState = c.lastState
c = ct
} }
//Count the failure and success //Count the failure and success
@ -129,31 +88,98 @@ func (c *counters) Count(statue State) {
c.ConsecutiveSuccesses++ c.ConsecutiveSuccesses++
} }
c.Requests++ c.Requests++
c.lastActivity = time.Now() //更新活动时间
c.lastState = statue c.lastState = statue
//fire event here
} }
//WrapperBreaker return a Wrapper to hold request ////////////////////////////////
func WrapperBreaker(c Circuit, failureThreshold uint32) Circuit { //way 1 对象式断路器
////////////////////////////////
//RequestBreaker for protection
type RequestBreaker struct {
options Options
mutex sync.Mutex
state State //断路器的当前状态
generation uint64
counts ICounter
}
//NewRequestBreaker return a breaker
func NewRequestBreaker(opts ...Option) *RequestBreaker {
defaultOptions := Options{
Name: "defaultBreakerName",
Expiry: time.Now().Add(time.Second * 20),
Interval: time.Second * 2, // interval to check status
Timeout: time.Second * 60, //default to 60 seconds
MaxRequests: 5,
WhenToBreak: func(counts counters) bool { return counts.ConsecutiveFailures > 2 },
OnStateChanged: func(name string, fromPre State, toCurrent State) {},
}
for _, setOption := range opts {
setOption(&defaultOptions)
}
return &RequestBreaker{
options: defaultOptions,
counts: &counters{},
generation: 0,
}
}
// Do the given requested work if the RequestBreaker accepts it.
// Do returns an error instantly if the RequestBreaker rejects the request.
// Otherwise, Execute returns the result of the request.
// If a panic occurs in the request, the RequestBreaker handles it as an error and causes the same panic again.
func (rb *RequestBreaker) Do(work func() (interface{}, error)) (interface{}, error) {
//before
fmt.Println("before do : request:", rb.counts.Total())
//do work from requested user
result, err := work()
fmt.Println("after do : request:", rb.counts.Total())
return result, err
}
////////////////////////////////
//way 2 简单的函数式断路器
////////////////////////////////
//Circuit of action stream,this is actually to do something.
//Circuit hold the really action
type Circuit func(context.Context) error
//Breaker return a closure wrapper to hold request,达到指定的失败次数后电路断开
func Breaker(c Circuit, failureThreshold uint32) Circuit {
//内部计数器 //内部计数器
cnt := counters{} cnt := counters{}
//ctx can be used hold parameters
return func(ctx context.Context) error { return func(ctx context.Context) error {
if cnt.ConsecutiveFailures >= failureThreshold { if cnt.ConsecutiveFailures >= failureThreshold {
canRetry := func(cnt counters) bool { canRetry := func(cnt counters) bool {
//间歇时间多个线程时候会存在同步文件需要lock操作
backoffLevel := cnt.ConsecutiveFailures - failureThreshold backoffLevel := cnt.ConsecutiveFailures - failureThreshold
// Calculates when should the circuit breaker resume propagating requests // Calculates when should the circuit breaker resume propagating requests
// to the service // to the service
shouldRetryAt := cnt.LastActivity().Add(time.Second * 2 << backoffLevel) backoffDuration := time.Second << backoffLevel
shouldRetryAt := cnt.LastActivity().Add(backoffDuration)
return time.Now().After(shouldRetryAt) return time.Now().After(shouldRetryAt)
} }
//如果仍然不能执行,直接返回失败
if !canRetry(cnt) { if !canRetry(cnt) {
// Fails fast instead of propagating requests to the circuit since // Fails fast instead of propagating requests to the circuit since
// not enough time has passed since the last failure to retry // not enough time has passed since the last failure to retry
@ -161,13 +187,17 @@ func WrapperBreaker(c Circuit, failureThreshold uint32) Circuit {
} }
} }
// 可以执行,则执行,并累计失败次数
// Unless the failure threshold is exceeded the wrapped service mimics the // Unless the failure threshold is exceeded the wrapped service mimics the
// old behavior and the difference in behavior is seen after consecutive failures // old behavior and the difference in behavior is seen after consecutive failures
// do the job
if err := c(ctx); err != nil { if err := c(ctx); err != nil {
//统计状态
cnt.Count(FailureState) cnt.Count(FailureState)
return err return err
} }
//统计成功状态
cnt.Count(SuccessState) cnt.Count(SuccessState)
return nil return nil
} }

View File

@ -3,12 +3,13 @@
* @Author: Edward * @Author: Edward
* @Date: 2020-05-11 10:55:28 * @Date: 2020-05-11 10:55:28
* @Last Modified by: Edward * @Last Modified by: Edward
* @Last Modified time: 2020-05-11 21:35:39 * @Last Modified time: 2020-05-21 14:08:53
*/ */
package circuit package circuit
import ( import (
"context"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
@ -19,15 +20,7 @@ var breaker *RequestBreaker
func TestBasicBreaker(t *testing.T) { func TestBasicBreaker(t *testing.T) {
readyToTrip := func(counts counters) bool { jobToDo := func() (interface{}, error) {
//失败率,可以由用户自己定义
failureRatio := float64(counts.TotalFailures) / float64(counts.Requests)
return counts.Requests >= 3 && failureRatio >= 0.6
}
breaker = NewRequestBreaker(Name("HTTP GET"), ReadyToTrip(readyToTrip))
body, err := breaker.Do(func() (interface{}, error) {
resp, err := http.Get("https://bing.com/robots.txt") resp, err := http.Get("https://bing.com/robots.txt")
if err != nil { if err != nil {
return nil, err return nil, err
@ -38,10 +31,47 @@ func TestBasicBreaker(t *testing.T) {
return nil, err return nil, err
} }
return body, nil return body, nil
}) }
whenCondition := func(counts counters) bool {
//失败率,可以由用户自己定义
failureRatio := float64(counts.TotalFailures) / float64(counts.Requests)
return counts.Requests >= 3 && failureRatio >= 0.6
}
breaker = NewRequestBreaker(Name("HTTP GET"), BreakIf(whenCondition))
body, err := breaker.Do(jobToDo)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
fmt.Println(string(body.([]byte))) fmt.Println(string(body.([]byte)))
} }
func TestFunctionalBreaker(t *testing.T) {
//something need to do
jobToDo := func(ctx context.Context) error {
resp, err := http.Get("https://bing.com/robots.txt")
if err != nil {
return err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
fmt.Println(string(body))
return nil
}
//wrapper and control job with a breaker
circuitWork := Breaker(jobToDo, 2 /* failureThreshold */)
params := context.TODO()
// do the job as usually
circuitWork(params)
}

View File

@ -176,6 +176,7 @@ func NewTwoStepCircuitBreaker(st Settings) *TwoStepCircuitBreaker {
const defaultInterval = time.Duration(0) * time.Second const defaultInterval = time.Duration(0) * time.Second
const defaultTimeout = time.Duration(60) * time.Second const defaultTimeout = time.Duration(60) * time.Second
//5 Consecutive Failures will break
func defaultReadyToTrip(counts Counts) bool { func defaultReadyToTrip(counts Counts) bool {
return counts.ConsecutiveFailures > 5 return counts.ConsecutiveFailures > 5
} }