Merge branch 'XX-3953/proxyErrorMetrics' into 'release'

XX-3953 / Proxy Error Metric (exponential moving average)

See merge request !29

Makefile

@@ -8,20 +8,20 @@ clean:
 	go mod vendor
 
 update:
-	-GOFLAGS="" go get -u all
+	-GOFLAGS="" go get all
 
 build:
 	go build ./...
 	go mod tidy
 
 update_release:
-	GOFLAGS="" go get -u gitlab.com/xx_network/primitives@release
-	GOFLAGS="" go get -u gitlab.com/xx_network/crypto@release
+	GOFLAGS="" go get gitlab.com/xx_network/primitives@release
+	GOFLAGS="" go get gitlab.com/xx_network/crypto@release
 
 update_master:
-	GOFLAGS="" go get -u gitlab.com/xx_network/primitives@master
-	GOFLAGS="" go get -u gitlab.com/xx_network/crypto@master
+	GOFLAGS="" go get gitlab.com/xx_network/primitives@master
+	GOFLAGS="" go get gitlab.com/xx_network/crypto@master
 
-master: clean update_master build
+master: update_master clean build
 
-release: clean update_release build
+release: update_release clean build

connect/host.go

@@ -17,6 +17,7 @@ import (
 	"gitlab.com/xx_network/comms/connect/token"
 	"gitlab.com/xx_network/crypto/signature/rsa"
 	tlsCreds "gitlab.com/xx_network/crypto/tls"
+	"gitlab.com/xx_network/primitives/exponential"
 	"gitlab.com/xx_network/primitives/id"
 	"gitlab.com/xx_network/primitives/rateLimiting"
 	"google.golang.org/grpc"
@@ -66,6 +67,10 @@ type Host struct {
 	// State tracking for host metric
 	metrics *Metric
 
+	// Tracks the exponential moving average of proxy error messages so that if
+	// too many connection error occur, the layer above can be informed
+	proxyErrorMetric *exponential.MovingAvg
+
 	coolOffBucket *rateLimiting.Bucket
 	inCoolOff     bool
 
@@ -89,6 +94,7 @@ func NewHost(id *id.ID, address string, cert []byte, params HostParams) (host *H
 		transmissionToken: token.NewLive(),
 		receptionToken:    token.NewLive(),
 		metrics:           newMetric(),
+		proxyErrorMetric:  exponential.NewMovingAvg(params.ProxyErrorMetricParams),
 		params:            params,
 		windowSize:        &windowSize,
 	}
@@ -272,6 +278,18 @@ func (h *Host) transmit(f func(conn *grpc.ClientConn) (interface{},
 		}
 	}
 
+	if err != nil {
+		// Check if the received error is a connection timeout and add it to the
+		// moving average. If the cutoff is reached for too many timeouts,
+		// return TooManyProxyError instead so that the host can be removed from
+		// the host pool on the layer above.
+		err2 := h.proxyErrorMetric.Intake(
+			exponential.BoolToFloat(strings.Contains(err.Error(), ProxyError)))
+		if err2 != nil {
+			err = errors.Errorf("%s: %+v", TooManyProxyError, err2)
+		}
+	}
+
 	return a, err
 }
 

connect/hostParams.go

@@ -8,6 +8,7 @@
 package connect
 
 import (
+	"gitlab.com/xx_network/primitives/exponential"
 	"google.golang.org/grpc/keepalive"
 	"time"
 )
@@ -51,6 +52,10 @@ type HostParams struct {
 
 	// KeepAlive Options for Host connections
 	KaClientOpts keepalive.ClientParameters
+
+	// ProxyErrorMetricParams are the parameters used for the proxy error
+	// tracker that uses exponential moving average (exponential.MovingAvg).
+	ProxyErrorMetricParams exponential.MovingAvgParams
 }
 
 // GetDefaultHostParams Get default set of host params
@@ -75,5 +80,6 @@ func GetDefaultHostParams() HostParams {
 			// For all connections, with and without streaming
 			PermitWithoutStream: true,
 		},
+		ProxyErrorMetricParams: exponential.DefaultMovingAvgParams(),
 	}
 }

connect/host_test.go

@@ -9,8 +9,12 @@ package connect
 
 import (
 	"bytes"
+	"github.com/pkg/errors"
+	jww "github.com/spf13/jwalterweatherman"
 	"gitlab.com/xx_network/primitives/id"
+	"google.golang.org/grpc"
 	"net"
+	"strings"
 	"testing"
 )
 
@@ -198,3 +202,42 @@ func TestHost_GetMetrics(t *testing.T) {
 	}
 
 }
+
+// Tests that Host.transmit returns the original proxy error when under the
+// proxy error threshold and returns TooManyProxyError when over the threshold.
+func TestHost_transmit_ProxyError(t *testing.T) {
+	jww.SetStdoutThreshold(jww.LevelTrace)
+	// Create the host
+	p := GetDefaultHostParams()
+	p.ProxyErrorMetricParams.Cutoff = 0.17
+	host, err := NewHost(&id.ID{}, "", nil, p)
+	if err != nil {
+		t.Fatalf("Unable to create host: %+v", host)
+	}
+	host.connection = &grpc.ClientConn{}
+
+	originalErr := errors.New("Unable to SendToAny via " +
+		"ZT9BlnUhZZaPGB/A0BBR6tIjRrASM5GcnXrSkepElWwB: Register: Failed " +
+		"requesting client key from gateway " +
+		"I3g/DVoWVGsz/JTh6DuccdgXT8o0fM+TtA21EppKPtcB: rpc error: code = " +
+		"Unknown desc = unable to connect to target host " +
+		"I3g/DVoWVGsz/JTh6DuccdgXT8o0fM+TtA21EppKPtcB..Did not replace host.")
+
+	f := func(*grpc.ClientConn) (interface{}, error) {
+		return nil, originalErr
+	}
+
+	// Check that the original error is returned when under the cutoff
+	_, err = host.transmit(f)
+	if err == nil || err != originalErr {
+		t.Errorf("Error did not contain expected message."+
+			"\nexpected: %s\nreceived: %+v", originalErr, err)
+	}
+
+	// Check that TooManyProxyError is returned when over the cutoff
+	_, err = host.transmit(f)
+	if err == nil || !strings.Contains(err.Error(), TooManyProxyError) {
+		t.Errorf("Error did not contain expected message."+
+			"\nexpected: %s\nreceived: %+v", TooManyProxyError, err)
+	}
+}

connect/trackedErrors.go

+///////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC                                          //
+//                                                                           //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file                                                              //
+///////////////////////////////////////////////////////////////////////////////
+
+package connect
+
+// This file contains errors that are tracked by or returned by the host
+// depending on their tracked metrics.
+
+// ProxyError is part of the error reported by gateway when host cannot be
+// reached. Its frequency is being tracked so that it can be excluded from the
+// host pool in the layer after a set number of occurrences.
+const ProxyError = "unable to connect to target host"
+
+// TooManyProxyError is the error returned instead of ProxyError, when it occurs
+// too many times.
+const TooManyProxyError = "too many proxy failures to target host"

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/jwalterweatherman v1.1.0
 	gitlab.com/xx_network/crypto v0.0.5-0.20220606200528-3f886fe49e81
-	gitlab.com/xx_network/primitives v0.0.4-0.20220324193139-b292d1ae6e7e
+	gitlab.com/xx_network/primitives v0.0.4-0.20220630163313-7890038258c6
 	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
 	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
 	google.golang.org/grpc v1.38.0

go.sum

@@ -83,8 +83,8 @@ gitlab.com/xx_network/primitives v0.0.0-20200803231956-9b192c57ea7c/go.mod h1:wt
 gitlab.com/xx_network/primitives v0.0.0-20200804183002-f99f7a7284da/go.mod h1:OK9xevzWCaPO7b1wiluVJGk7R5ZsuC7pHY5hteZFQug=
 gitlab.com/xx_network/primitives v0.0.2/go.mod h1:cs0QlFpdMDI6lAo61lDRH2JZz+3aVkHy+QogOB6F/qc=
 gitlab.com/xx_network/primitives v0.0.4-0.20220222211843-901fa4a2d72b/go.mod h1:9imZHvYwNFobxueSvVtHneZLk9wTK7HQTzxPm+zhFhE=
-gitlab.com/xx_network/primitives v0.0.4-0.20220324193139-b292d1ae6e7e h1:F651DdbU9n5qOLN8rdyAIluXWtm5Wl4jwH5D6ED3bSI=
-gitlab.com/xx_network/primitives v0.0.4-0.20220324193139-b292d1ae6e7e/go.mod h1:AXVVFt7dDAeIUpOGPiStCcUIKsBXLWbmV/BgZ4T+tOo=
+gitlab.com/xx_network/primitives v0.0.4-0.20220630163313-7890038258c6 h1:3It6ILDHn/9J/Oi7MfMjkidKPe7vbFCy5JQtXx8EfYM=
+gitlab.com/xx_network/primitives v0.0.4-0.20220630163313-7890038258c6/go.mod h1:AXVVFt7dDAeIUpOGPiStCcUIKsBXLWbmV/BgZ4T+tOo=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=