Return rate limiting error up the stack

go.mod

@@ -18,7 +18,7 @@ require (
 	gitlab.com/elixxir/primitives v0.0.3-0.20220104173924-275cb9d7834f
 	gitlab.com/xx_network/comms v0.0.4-0.20220126231737-fe2338016cce
 	gitlab.com/xx_network/crypto v0.0.5-0.20211227194420-f311e8920467
-	gitlab.com/xx_network/primitives v0.0.4-0.20211222205802-03e9d7d835b0
+	gitlab.com/xx_network/primitives v0.0.4-0.20220214225457-636c7b31cb78
 	golang.org/x/crypto v0.0.0-20220128200615-198e4374d7ed
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
 	google.golang.org/grpc v1.42.0

go.sum

@@ -299,6 +299,8 @@ gitlab.com/xx_network/primitives v0.0.0-20200804183002-f99f7a7284da/go.mod h1:OK
 gitlab.com/xx_network/primitives v0.0.2/go.mod h1:cs0QlFpdMDI6lAo61lDRH2JZz+3aVkHy+QogOB6F/qc=
 gitlab.com/xx_network/primitives v0.0.4-0.20211222205802-03e9d7d835b0 h1:IHHb59DJEKk02HgfxddqK2ilvkRMAUdPIBFn8rmjjIg=
 gitlab.com/xx_network/primitives v0.0.4-0.20211222205802-03e9d7d835b0/go.mod h1:9imZHvYwNFobxueSvVtHneZLk9wTK7HQTzxPm+zhFhE=
+gitlab.com/xx_network/primitives v0.0.4-0.20220214225457-636c7b31cb78 h1:ZtX91rUOadX+Z46ATb0iJLFNOiGy6RqegEyzmqlYis8=
+gitlab.com/xx_network/primitives v0.0.4-0.20220214225457-636c7b31cb78/go.mod h1:9imZHvYwNFobxueSvVtHneZLk9wTK7HQTzxPm+zhFhE=
 gitlab.com/xx_network/ring v0.0.3-0.20210527191221-ce3f170aabd5 h1:FY+4Rh1Q2rgLyv10aKJjhWApuKRCR/054XhreudfAvw=
 gitlab.com/xx_network/ring v0.0.3-0.20210527191221-ce3f170aabd5/go.mod h1:aLzpP2TiZTQut/PVHR40EJAomzugDdHXetbieRClXIM=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=

network/message/sendCmix.go

@@ -27,6 +27,8 @@ import (
 	"gitlab.com/xx_network/primitives/id"
 	"gitlab.com/xx_network/primitives/id/ephemeral"
 	"gitlab.com/xx_network/primitives/netTime"
+	"gitlab.com/xx_network/primitives/rateLimiting"
+	"strings"
 	"time"
 )
 
@@ -117,7 +119,7 @@ func sendCmixHelper(sender *gateway.Sender, msg format.Message,
 		remainingTime := cmixParams.Timeout - elapsed
 		bestRound, err := instance.GetWaitingRounds().GetUpcomingRealtime(remainingTime, attempted, sendTimeBuffer)
 		if err != nil {
-			jww.WARN.Printf("[SendCMIX-%s] Failed to GetUpcomingRealtime " +
+			jww.WARN.Printf("[SendCMIX-%s] Failed to GetUpcomingRealtime "+
 				"(msgDigest: %s): %+v", cmixParams.DebugTag, msg.Digest(), err)
 		}
 		if bestRound == nil {
@@ -139,7 +141,7 @@ func sendCmixHelper(sender *gateway.Sender, msg format.Message,
 			}
 		}
 		if containsBlacklisted {
-			jww.WARN.Printf("[SendCMIX-%s]Round %d contains blacklisted node, " +
+			jww.WARN.Printf("[SendCMIX-%s]Round %d contains blacklisted node, "+
 				"skipping...", cmixParams.DebugTag, bestRound.ID)
 			continue
 		}
@@ -147,7 +149,7 @@ func sendCmixHelper(sender *gateway.Sender, msg format.Message,
 		// Retrieve host and key information from round
 		firstGateway, roundKeys, err := processRound(instance, session, nodeRegistration, bestRound, recipient.String(), msg.Digest())
 		if err != nil {
-			jww.WARN.Printf("[SendCMIX-%s]SendCmix failed to process round" +
+			jww.WARN.Printf("[SendCMIX-%s]SendCmix failed to process round"+
 				" (will retry): %v", cmixParams.DebugTag, err)
 			continue
 		}
@@ -165,7 +167,7 @@ func sendCmixHelper(sender *gateway.Sender, msg format.Message,
 
 		jww.INFO.Printf("[SendCMIX-%s] Sending to EphID %d (%s), "+
 			"on round %d (msgDigest: %s, ecrMsgDigest: %s) "+
-			"via gateway %s",  cmixParams.DebugTag,
+			"via gateway %s", cmixParams.DebugTag,
 			ephID.Int64(), recipient, bestRound.ID, msg.Digest(),
 			encMsg.Digest(), firstGateway.String())
 
@@ -213,6 +215,13 @@ func sendCmixHelper(sender *gateway.Sender, msg format.Message,
 
 		// if the comm errors or the message fails to send, continue retrying.
 		if err != nil {
+			if strings.Contains(err.Error(), rateLimiting.ClientRateLimitErr) {
+				jww.ERROR.Printf("[SendCMIX-%s] SendCmix failed to send to EphID %d (%s) on "+
+					"round %d: %+v", cmixParams.DebugTag, ephID.Int64(), recipient,
+					bestRound.ID, err)
+				return 0, ephemeral.Id{}, err
+			}
+
 			jww.ERROR.Printf("[SendCMIX-%s] SendCmix failed to send to EphID %d (%s) on "+
 				"round %d, trying a new round: %+v", cmixParams.DebugTag, ephID.Int64(), recipient,
 				bestRound.ID, err)