From ec3b00c48426a045339eaccbf5f51e5bea5580c8 Mon Sep 17 00:00:00 2001 From: Dong Chen Date: Wed, 9 Dec 2015 17:14:56 -0800 Subject: [PATCH] Reorganize engine failure detection procedure. Change engine option 'RefreshRetry' to 'FailureRetry'. Signed-off-by: Dong Chen --- cli/commands.go | 2 +- cli/flags.go | 6 ++-- cli/manage.go | 8 ++--- cluster/engine.go | 45 +++++++++++++--------------- cluster/engine_test.go | 4 +-- cluster/mesos/cluster_test.go | 2 +- cluster/swarm/cluster_test.go | 2 +- test/integration/engine_options.bats | 4 +-- 8 files changed, 34 insertions(+), 39 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index 73790935c9..6e9e6bb9f3 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -26,7 +26,7 @@ var ( flHosts, flLeaderElection, flLeaderTTL, flManageAdvertise, flTLS, flTLSCaCert, flTLSCert, flTLSKey, flTLSVerify, - flRefreshIntervalMin, flRefreshIntervalMax, flRefreshRetry, + flRefreshIntervalMin, flRefreshIntervalMax, flFailureRetry, flHeartBeat, flEnableCors, flCluster, flDiscoveryOpt, flClusterOpt}, diff --git a/cli/flags.go b/cli/flags.go index 6d783189e9..b4f646c935 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -71,10 +71,10 @@ var ( Value: "60s", Usage: "set engine refresh maximum interval", } - flRefreshRetry = cli.IntFlag{ - Name: "engine-refresh-retry", + flFailureRetry = cli.IntFlag{ + Name: "engine-failure-retry", Value: 3, - Usage: "set engine refresh retry count on failure", + Usage: "set engine failure retry count", } flEnableCors = cli.BoolFlag{ Name: "api-enable-cors, cors", diff --git a/cli/manage.go b/cli/manage.go index 260cc8bc67..a6ed522176 100644 --- a/cli/manage.go +++ b/cli/manage.go @@ -240,14 +240,14 @@ func manage(c *cli.Context) { if refreshMaxInterval < refreshMinInterval { log.Fatal("max refresh interval cannot be less than min refresh interval") } - refreshRetry := c.Int("engine-refresh-retry") - if refreshRetry <= 0 { - log.Fatal("invalid refresh retry count") + failureRetry := c.Int("engine-failure-retry") + if failureRetry <= 0 { + log.Fatal("invalid failure retry count") } engineOpts := &cluster.EngineOpts{ RefreshMinInterval: refreshMinInterval, RefreshMaxInterval: refreshMaxInterval, - RefreshRetry: refreshRetry, + FailureRetry: failureRetry, } uri := getDiscovery(c) diff --git a/cluster/engine.go b/cluster/engine.go index 1bba83fde5..7f8c01cb08 100644 --- a/cluster/engine.go +++ b/cluster/engine.go @@ -24,9 +24,6 @@ const ( // Minimum docker engine version supported by swarm. minSupportedVersion = version.Version("1.6.0") - - // Engine failureCount threshold - engineFailureCountThreshold = 3 ) // delayer offers a simple API to random delay within a given time range. @@ -62,7 +59,7 @@ func (d *delayer) Wait() <-chan time.Time { type EngineOpts struct { RefreshMinInterval time.Duration RefreshMaxInterval time.Duration - RefreshRetry int + FailureRetry int } // Engine represents a docker engine @@ -86,7 +83,7 @@ type Engine struct { client dockerclient.Client eventHandler EventHandler healthy bool - failureCount int64 + failureCount int overcommitRatio int64 opts *EngineOpts } @@ -188,6 +185,17 @@ func (e *Engine) IsHealthy() bool { return e.healthy } +// SetHealthy sets engine healthy state +func (e *Engine) SetHealthy(state bool) { + e.Lock() + e.healthy = state + // if engine is healthy, clear failureCount + if state { + e.failureCount = 0 + } + e.Unlock() +} + // Status returns the health status of the Engine: Healthy or Unhealthy func (e *Engine) Status() string { if e.healthy { @@ -200,23 +208,12 @@ func (e *Engine) Status() string { func (e *Engine) IncFailureCount() { e.Lock() e.failureCount++ - if e.healthy && e.failureCount >= engineFailureCountThreshold { + if e.healthy && e.failureCount >= e.opts.FailureRetry { e.healthy = false } e.Unlock() } -// SetEngineHealth sets engine healthy state -func (e *Engine) SetEngineHealth(state bool) { - e.Lock() - e.healthy = state - // if engine is healthy, clear failureCount - if state { - e.failureCount = 0 - } - e.Unlock() -} - // Gather engine specs (CPU, memory, constraints, ...). func (e *Engine) updateSpecs() error { info, err := e.client.Info() @@ -435,7 +432,6 @@ func (e *Engine) updateContainer(c dockerclient.Container, containers map[string } func (e *Engine) refreshLoop() { - failedAttempts := 0 for { var err error @@ -456,15 +452,15 @@ func (e *Engine) refreshLoop() { } if err != nil { - failedAttempts++ - if failedAttempts >= e.opts.RefreshRetry && e.healthy { + e.failureCount++ + if e.failureCount >= e.opts.FailureRetry && e.healthy { e.emitEvent("engine_disconnect") - e.SetEngineHealth(false) - log.WithFields(log.Fields{"name": e.Name, "id": e.ID}).Errorf("Flagging engine as dead. Updated state failed %d times: %v", failedAttempts, err) + e.SetHealthy(false) + log.WithFields(log.Fields{"name": e.Name, "id": e.ID}).Errorf("Flagging engine as dead. Updated state failed %d times: %v", e.failureCount, err) } } else { if !e.healthy { - log.WithFields(log.Fields{"name": e.Name, "id": e.ID}).Infof("Engine came back to life after %d retries. Hooray!", failedAttempts) + log.WithFields(log.Fields{"name": e.Name, "id": e.ID}).Infof("Engine came back to life after %d retries. Hooray!", e.failureCount) if err := e.updateSpecs(); err != nil { log.WithFields(log.Fields{"name": e.Name, "id": e.ID}).Errorf("Update engine specs failed: %v", err) continue @@ -473,8 +469,7 @@ func (e *Engine) refreshLoop() { e.client.StartMonitorEvents(e.handler, nil) e.emitEvent("engine_reconnect") } - e.SetEngineHealth(true) - failedAttempts = 0 + e.SetHealthy(true) } } } diff --git a/cluster/engine_test.go b/cluster/engine_test.go index 31941142e1..6552c6eba7 100644 --- a/cluster/engine_test.go +++ b/cluster/engine_test.go @@ -34,13 +34,13 @@ var ( engOpts = &EngineOpts{ RefreshMinInterval: time.Duration(30) * time.Second, RefreshMaxInterval: time.Duration(60) * time.Second, - RefreshRetry: 3, + FailureRetry: 3, } ) func TestEngineFailureCount(t *testing.T) { engine := NewEngine("test", 0, engOpts) - for i := 0; i < engineFailureCountThreshold; i++ { + for i := 0; i < engine.opts.FailureRetry; i++ { assert.True(t, engine.IsHealthy()) engine.IncFailureCount() } diff --git a/cluster/mesos/cluster_test.go b/cluster/mesos/cluster_test.go index c05bf66d96..1ce8116eb5 100644 --- a/cluster/mesos/cluster_test.go +++ b/cluster/mesos/cluster_test.go @@ -13,7 +13,7 @@ func createAgent(t *testing.T, ID string, containers ...*cluster.Container) *age engOpts := &cluster.EngineOpts{ RefreshMinInterval: time.Duration(30) * time.Second, RefreshMaxInterval: time.Duration(60) * time.Second, - RefreshRetry: 3, + FailureRetry: 3, } engine := cluster.NewEngine(ID, 0, engOpts) engine.Name = ID diff --git a/cluster/swarm/cluster_test.go b/cluster/swarm/cluster_test.go index 1d3c04e19a..bb02d2918d 100644 --- a/cluster/swarm/cluster_test.go +++ b/cluster/swarm/cluster_test.go @@ -43,7 +43,7 @@ var ( engOpts = &cluster.EngineOpts{ RefreshMinInterval: time.Duration(30) * time.Second, RefreshMaxInterval: time.Duration(60) * time.Second, - RefreshRetry: 3, + FailureRetry: 3, } ) diff --git a/test/integration/engine_options.bats b/test/integration/engine_options.bats index c2347489ab..aef39fde29 100644 --- a/test/integration/engine_options.bats +++ b/test/integration/engine_options.bats @@ -14,7 +14,7 @@ load helpers [[ "${output}" == *"max refresh interval cannot be less than min refresh interval"* ]] # engine refresh retry count - run swarm manage --engine-refresh-retry 0 --advertise 127.0.0.1:$SWARM_BASE_PORT 192.168.56.202:4444 + run swarm manage --engine-failure-retry 0 --advertise 127.0.0.1:$SWARM_BASE_PORT 192.168.56.202:4444 [ "$status" -ne 0 ] - [[ "${output}" == *"invalid refresh retry count"* ]] + [[ "${output}" == *"invalid failure retry count"* ]] }