Merge branch 'projects/crust_RELEASE' into 'release'

Projects/crust release

See merge request !40

basice2e/crust.sh

+#!/bin/bash
+
+# This file contains logic to run clients with the specific goal of putting
+# backups testing backups on Crust's architecture. These clients will then
+# recover these backups and ensure the recovered file matches the originally
+# backed up file.
+
+# NOTE: This is verbose on purpose.
+################################################################################
+## Initial Set Up & Clean Up of Past Runs
+################################################################################
+
+set -e
+rm -fr results.bak || true
+mv results results.bak || rm -fr results || true
+rm *-contact.json || true
+rm server-5.qdstrm || true
+rm server-5.qdrep || true
+
+mkdir -p .elixxir
+
+if [ $# -gt 1 ]
+then
+    echo "usage: $0 [gatewayip:port]"
+    exit
+fi
+
+
+NETWORKENTRYPOINT=$1
+
+DEBUGLEVEL=${DEBUGLEVEL-1}
+
+SERVERLOGS=results/servers
+GATEWAYLOGS=results/gateways
+UDBOUT=results/udb-console.txt
+CLIENTOUT=results/clients
+CLIENTCLEAN=results/clients-cleaned
+
+CLIENTOPTS="--password hello --ndf results/ndf.json --verify-sends --sendDelay 100 --waitTimeout 360 -v $DEBUGLEVEL"
+CLIENTUDOPTS="--password hello --ndf results/ndf.json -v $DEBUGLEVEL"
+CLIENTALTUDOPTS="--alternateUd --altUdCert crustUd.crt --altUdContactFile crustUdContact.bin --altUdAddress 18.198.117.203:11420"
+
+# FIXME | As it stands, this needs to be incremented every run. This is so that
+# FIXME | the client is registered w/ UD used for testing with a unique username.
+# FIXME | Ideally there would be a mechanism to make this repeatable (ie reuse
+# FIXME | the username), and the (commented out) removeUser function attempted
+# FIXME | to do that by  removing the user from UD at the end of every run.
+# FIXME | This is accomplished by trap (Bash's equivalent of defer). However,
+# FIXME | the function never successfully removes the user. There should be a way
+# FIXME | to fix this, but incrementation is left until this is resolved.
+# FIXME | Alternative solutions include registering only if a blob file does not
+# FIXME | already exist for the client.
+CLIENTID=352
+ACCOUNTNAME=crustIntegrationTest$CLIENTID
+echo "CLIENTID - $CLIENTID"
+echo "ACCOUNTNAME - $ACCOUNTNAME"
+echo "Note that these values must be incremented each run (see the FIXME)"
+
+
+mkdir -p $SERVERLOGS
+mkdir -p $GATEWAYLOGS
+mkdir -p $CLIENTOUT
+mkdir -p $CLIENTCLEAN
+
+################################################################################
+## Network Set Up
+################################################################################
+
+# removeUser will remove the user from the UD
+removeUser() {
+  CLIENTCMD="timeout 240s ../bin/client ud $CLIENTUDOPTS $CLIENTALTUDOPTS -l $CLIENTOUT/client$CLIENTID.log -s blob$CLIENTID --remove $ACCOUNTNAME"
+  eval $CLIENTCMD >> $CLIENTOUT/client$CLIENTID.txt &
+  PIDVAL=$!
+  echo "$CLIENTCMD -- $PIDVAL"
+  wait $PIDVAL
+
+}
+
+
+# Ensure removeUser is called whenever this script closes, on success or failure
+#trap removeUser EXIT
+#trap removeUser INT
+
+
+if [ "$NETWORKENTRYPOINT" == "betanet" ]
+then
+    NETWORKENTRYPOINT=$(sort -R betanet.txt | head -1)
+elif [ "$NETWORKENTRYPOINT" == "mainnet" ]
+then
+    NETWORKENTRYPOINT=$(sort -R mainnet.txt | head -1)
+elif [ "$NETWORKENTRYPOINT" == "release" ]
+then
+    NETWORKENTRYPOINT=$(sort -R release.txt | head -1)
+elif [ "$NETWORKENTRYPOINT" == "devnet" ]
+then
+    NETWORKENTRYPOINT=$(sort -R devnet.txt | head -1)
+elif [ "$NETWORKENTRYPOINT" == "" ]
+then
+    NETWORKENTRYPOINT=$(head -1 network.config)
+fi
+
+echo "NETWORK: $NETWORKENTRYPOINT"
+
+if [ "$NETWORKENTRYPOINT" == "localhost:8440" ]
+then
+    source network.sh
+
+else
+    echo "Connecting to network defined at $NETWORKENTRYPOINT"
+    echo $NETWORKENTRYPOINT > results/startgwserver.txt
+fi
+
+echo "DOWNLOADING TLS Cert..."
+CMD="openssl s_client -showcerts -connect $(tr -d '[:space:]' < results/startgwserver.txt)"
+echo $CMD
+eval $CMD < /dev/null 2>&1 > "results/startgwcert.bin"
+CMD="cat results/startgwcert.bin | openssl x509 -outform PEM"
+echo $CMD
+eval $CMD > "results/startgwcert.pem"
+head "results/startgwcert.pem"
+
+echo "DOWNLOADING NDF..."
+CLIENTCMD="../bin/client getndf --gwhost $(tr -d '[:space:]' < results/startgwserver.txt) --cert results/startgwcert.pem"
+eval $CLIENTCMD >> results/ndf.json 2>&1 &
+PIDVAL=$!
+echo "$CLIENTCMD -- $PIDVAL"
+wait $PIDVAL
+
+cat results/ndf.json | jq . | head -5
+
+file results/ndf.json
+
+if [ ! -s results/ndf.json ]
+then
+    echo "results/ndf.json is empty, cannot proceed"
+    exit -1
+fi
+
+
+#export GRPC_GO_LOG_VERBOSITY_LEVEL=99
+#export GRPC_GO_LOG_SEVERITY_LEVEL=info
+
+###############################################################################
+# Test Crust
+###############################################################################
+
+echo "TESTING CRUST..."
+
+# Register username with UD
+CLIENTCMD="timeout 240s ../bin/client ud $CLIENTUDOPTS $CLIENTALTUDOPTS -l $CLIENTOUT/client$CLIENTID.log -s blob$CLIENTID --register $ACCOUNTNAME"
+eval $CLIENTCMD >> $CLIENTOUT/client$CLIENTID.txt &
+PIDVAL=$!
+echo "$CLIENTCMD -- $PIDVAL"
+wait $PIDVAL
+
+# Upload file to Crust
+CLIENTCMD="timeout 240s ../bin/client crust $CLIENTUDOPTS $CLIENTALTUDOPTS -l $CLIENTOUT/client$CLIENTID.log -s blob$CLIENTID --upload --file LoremIpsum.txt"
+eval $CLIENTCMD >> $CLIENTOUT/client$CLIENTID.txt &
+PIDVAL=$!
+echo "$CLIENTCMD -- $PIDVAL"
+wait $PIDVAL
+
+# Recover file from Crust
+CLIENTCMD="timeout 240s ../bin/client crust $CLIENTUDOPTS $CLIENTALTUDOPTS -l $CLIENTOUT/client$CLIENTID.log -s blob$CLIENTID --recover --file LoremIpsum.txt"
+eval $CLIENTCMD >> $CLIENTOUT/client$CLIENTID.txt &
+PIDVAL=$!
+echo "$CLIENTCMD -- $PIDVAL"
+wait $PIDVAL
+
+###############################################################################
+# Close Local Network
+###############################################################################
+
+if [ "$NETWORKENTRYPOINT" == "localhost:8440" ]
+then
+    cat $SERVERLOGS/server-*.log | grep -a "ERROR" | grep -a -v "context" | grep -av "metrics" | grep -av "database" | grep -av RequestClientKey > results/server-errors.txt || true
+    cat $SERVERLOGS/server-*.log | grep -a "FATAL" | grep -a -v "context" | grep -av "transport is closing" | grep -av "database" >> results/server-errors.txt || true
+    diff -aruN results/server-errors.txt noerrors.txt
+    IGNOREMSG="GetRoundBufferInfo: Error received: rpc error: code = Unknown desc = round buffer is empty"
+    cat $GATEWAYLOGS/*.log | grep -a "ERROR" | grep -av "context" | grep -av "certificate" | grep -av "Failed to read key" | grep -av "$IGNOREMSG" > results/gateway-errors.txt || true
+    cat $GATEWAYLOGS/*.log | grep -a "FATAL" | grep -av "context" | grep -av "transport is closing" >> results/gateway-errors.txt || true
+    diff -aruN results/gateway-errors.txt noerrors.txt
+fi
\ No newline at end of file

basice2e/crustUd.crt

+-----BEGIN CERTIFICATE-----
+MIIFqTCCA5GgAwIBAgIUEVshfaLgjzuVfLlyk0mTsXIkVewwDQYJKoZIhvcNAQEL
+BQAwgYAxCzAJBgNVBAYTAktZMRQwEgYDVQQHDAtHZW9yZ2UgVG93bjETMBEGA1UE
+CgwKeHggbmV0d29yazEPMA0GA1UECwwGRGV2T3BzMRMwEQYDVQQDDAp4eC5uZXR3
+b3JrMSAwHgYJKoZIhvcNAQkBFhFhZG1pbnNAeHgubmV0d29yazAeFw0yMTA1MDcx
+NTExNTVaFw0zMTA1MDUxNTExNTVaMIGAMQswCQYDVQQGEwJLWTEUMBIGA1UEBwwL
+R2VvcmdlIFRvd24xEzARBgNVBAoMCnh4IG5ldHdvcmsxDzANBgNVBAsMBkRldk9w
+czETMBEGA1UEAwwKeHgubmV0d29yazEgMB4GCSqGSIb3DQEJARYRYWRtaW5zQHh4
+Lm5ldHdvcmswggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC1dYoVYmYA
+L4TaKC9g3cTM8LqTD7Ofdo4LKkkluqPKadX6rZyrvO90LtOtcH/1yxipTOnaKF0J
+9/kTrxgTroKgIrHuzHOE7w87hcV4zPveYX+4nc8OBQRkVh0slIfPGUWQMSAx0ojv
+Aqstkrk8SapcSJK0AJi2wHephJhSnmnKbq56NSD4VwGbT9O8YAqsrU0FzGxTcers
+QILgQY3BdcnzJL3VRjzMaLT3oCbXS2FPD6Z46JVs/jCc+fg7TT7N1VJwsNJ0w7gR
+nV3iB5XG/9Em/khQ2Ne2mPRoh4xaXPFeAmXYhiREFI2t4LrgeHwySuRhsRLolYA+
+95nSZ1FCuPabV+jv2UURLN581VzLjgOA8Ad60P1yKfyP6N2BcR/99xD/Y+pBvsn5
+wgSVbPTxq15M3WehPLVvoie66pg25uLo+EdyvkzFIBjdy8oNW5qKB6IhrqM5OKsp
+xvdh/nQ/6bikIZxmdJqhTs0slhD6NlY+RnHNg/9xyDU/aoU/tn06cL9xlhVA88A/
+I1Tz5JJdF6G05O/MaTe5FaCJJly6+uY3R6O9f4eKdeuwVdvLHjKvFHir78djG5wq
+xYieGaiUSxy6kr0b283pVNxWlWVPUxlRQpr9EWB9cfKzjuMie83xdCGfJOgfiw1q
+KOirdiO3Y/goSsndolKrARWRO5Af4P4f/wIDAQABoxkwFzAVBgNVHREEDjAMggp4
+eC5uZXR3b3JrMA0GCSqGSIb3DQEBCwUAA4ICAQAIUMML1m+NXFF2qnXrkbJUrr0Y
+kMejzFkSrx0PypqXThDUwdCdkYZkNzcfTB/jvNMn6PfU7uR5OYqTgKjHyGxG7ZZs
+RhxL21jTUYq9H45aMRYJzOZ/jOag2QbrV+46YsbCMJoZhD9da2qVOLNPHyG1pYfY
+HzlObq+uy23iNQSL9dNSw9PbvPY8UVMqkY9dXEqFtOuivLoacUU6+ZiOnp/Cbed8
+E+oIhP7U3Zlnl1WMyMM94ltdVOqpzyiaNuqLE1S7ngN1qukiu63B50JH7YCxiwuC
+IrcyRErwrCPFdpjxsgJViH3cQG3Y07Eof2KJRYoyJ1NF9UgXfTpZ3P7lDSEtDre2
+YLKZbAPJRYwnyrPmw/Gs1PNIf02C+UglgqhtcpvTLQ+PxJp0u58JJfKqUDqzOAYl
+OtVJJ71coADrz9ON1W9TsJ154sHIQVF/wjLic9mUOnn0zsWuk2ClcGJp67UF8ZSe
+eWbnQipc/rxcsh9KF/n+lG7thmEaJ10tntr0X5vYR4e1FjIlfc2wk46yH7+pIl/3
+R9quX/3zRxxNKNugkz+Jp+WeL6oP2ozzh4Nr9H1rdAsgkj89t2RKteXp4XZ8Pp1i
+uXO9ulyHkAvQxyehoFk44Q0Z2AEkSDbNCtQ5yRKTKoeeL04vbAvn7L0XWQ7jDJje
+xHGnyrYJGhiR8Sp20w==
+-----END CERTIFICATE-----

basice2e/crustUdContact.bin

+<xxc(2)LF2ccT+sdqh0AIKlFFeDOJdnxzbQQYhGStgxhOXmijIDkAZiB9kZo+Dl3bRSbBi5pXZ82rOu2IQXz9+5sspChvoccZqgC/dXGhlesmiNy/EbKxWtptTF4tcNyQxtnmCXg1p/HwKey4G2XDekTw86lq6Lpmj72jozvRWlQisqvWz/5deiPaeFGKDKC0OrrDFnIib7WnKqdYt4XyTKdmObnmbvdCbliZq0zBl7J40qKy5FypYXGlZjStIm0R1qtD4XHMZMsrMJEGxdM55zJdSzknXbR8MNahUrGMyUOTivXLHzojYLht0gFQifKMVWhrDjUoVQV43KOLPmdBwY/2Kc5KvVloDeuDXYY0i7tD63gNIp9JA3gJQUJymDdwqbS13riT1DMHHkdTzKEyGdHS+v2l7AVSlJBiTKuyM00FBNuXhhIcFR7ONFCf8cRPOPPBx3Q6iHNsvsca3KPNhwOJBgaQvHSkjIMsudiR954QbwG9rbi2vxVobIgWYMl5j6vlBS/9rfbE/uLdTEQZfNsLKDCIVCCI4I1bYZxZrDLPrfXTrN6W0sCLE7a/kRBQAAAgA7+LwJqiv9O1ogLnS4TYkSEg==xxc>
\ No newline at end of file