run.sh

#!/bin/bash

# NOTE: This is verbose on purpose.

set -e
rm -fr results.bak || true
mv results results.bak || rm -fr results || true
rm -fr blob* || true
rm *-contact.json || true
rm server-5.qdstrm || true
rm server-5.qdrep || true

mkdir -p .elixxir

if [ $# -gt 1 ]
then
    echo "usage: $0 [gatewayip:port]"
    exit
fi


NETWORKENTRYPOINT=$1

DEBUGLEVEL=${DEBUGLEVEL-1}


SERVERLOGS=results/servers
GATEWAYLOGS=results/gateways
CLIENTOUT=results/clients
UDBOUT=results/udb-console.txt
CLIENTCLEAN=results/clients-cleaned

CLIENTOPTS="--password hello --ndf results/ndf.json --verify-sends --sendDelay 100 --waitTimeout 360 --unsafe-channel-creation -v $DEBUGLEVEL"
CLIENTUDOPTS="--password hello --ndf results/ndf.json -v $DEBUGLEVEL"
CLIENTSINGLEOPTS="--password hello --waitTimeout 360 --ndf results/ndf.json -v $DEBUGLEVEL"
CLIENTGROUPOPTS="--password hello --waitTimeout 360 --ndf results/ndf.json -v $DEBUGLEVEL"
CLIENTFILETRANSFEROPTS="--password hello --waitTimeout 360 --ndf results/ndf.json -v $DEBUGLEVEL"
CLIENTREKEYOPTS="--password hello --ndf results/ndf.json --verify-sends --waitTimeout 420 --unsafe-channel-creation -v $DEBUGLEVEL"
CLIENTBACKUPOPTS="--password hello --ndf results/ndf.json -v $DEBUGLEVEL"

mkdir -p $SERVERLOGS
mkdir -p $GATEWAYLOGS
mkdir -p $CLIENTOUT
mkdir -p $CLIENTCLEAN

if [ "$NETWORKENTRYPOINT" == "betanet" ]
then
    NETWORKENTRYPOINT=$(sort -R betanet.txt | head -1)
elif [ "$NETWORKENTRYPOINT" == "mainnet" ]
then
    NETWORKENTRYPOINT=$(sort -R mainnet.txt | head -1)
elif [ "$NETWORKENTRYPOINT" == "release" ]
then
    NETWORKENTRYPOINT=$(sort -R release.txt | head -1)
elif [ "$NETWORKENTRYPOINT" == "devnet" ]
then
    NETWORKENTRYPOINT=$(sort -R devnet.txt | head -1)
elif [ "$NETWORKENTRYPOINT" == "" ]
then
    NETWORKENTRYPOINT=$(head -1 network.config)
fi

echo "NETWORK: $NETWORKENTRYPOINT"

if [ "$NETWORKENTRYPOINT" == "localhost:8440" ]
then
    echo "STARTING SERVERS..."

    # Copy udbContact into place when running locally.
    cp udbContact.bin results/udbContact.bin

    PERMCMD="../bin/permissioning --logLevel $DEBUGLEVEL -c permissioning.yaml "
    $PERMCMD > results/permissioning-console.txt 2>&1 &
    PIDVAL=$!
    echo "$PERMCMD -- $PIDVAL"


    # Run Client Registrar
    CLIENT_REG_CMD="../bin/client-registrar \
    -l 2 -c client-registrar.yaml"
    $CLIENT_REG_CMD > results/client-registrat-console.txt 2>&1 &
    PIDVAL=$!
    echo "$CLIENT_REG_CMD -- $PIDVAL"

    for SERVERID in $(seq 5 -1 1)
    do
        IDX=$(($SERVERID - 1))
        SERVERCMD="../bin/server --logLevel $DEBUGLEVEL --config server-$SERVERID.yaml"
        if [ $SERVERID -eq 5 ] && [ -n "$NSYSENABLED" ]
        then
            SERVERCMD="nsys profile --session-new=gputest --trace=cuda -o server-$SERVERID $SERVERCMD"
        fi
        $SERVERCMD > $SERVERLOGS/server-$SERVERID-console.txt 2>&1 &
        PIDVAL=$!
        echo "$SERVERCMD -- $PIDVAL"
    done

    # Start gateways
    for GWID in $(seq 5 -1 1)
    do
        IDX=$(($GWID - 1))
        GATEWAYCMD="../bin/gateway --logLevel $DEBUGLEVEL --config gateway-$GWID.yaml"
        $GATEWAYCMD > $GATEWAYLOGS/gateway-$GWID-console.txt 2>&1 &
        PIDVAL=$!
        echo "$GATEWAYCMD -- $PIDVAL"
    done

    jobs -p > results/serverpids

    finish() {
        echo "STOPPING SERVERS AND GATEWAYS..."
        if [ -n "$NSYSENABLED" ]
        then
            nsys stop --session=gputest
        fi
        # NOTE: jobs -p doesn't work in a signal handler
        for job in $(cat results/serverpids)
        do
            echo "KILLING $job"
            kill $job || true
        done

        sleep 5

        for job in $(cat results/serverpids)
        do
            echo "KILL -9 $job"
            kill -9 $job || true
        done
        #tail $SERVERLOGS/*
        #tail $CLIENTCLEAN/*
        #diff -aruN clients.goldoutput $CLIENTCLEAN
    }

    trap finish EXIT
    trap finish INT

    # Sleeps can die in a fire on the sun, we wait for the servers to start running
    # rounds
    rm rid.txt || true
    touch rid.txt
    cnt=0
    echo -n "Waiting for a round to run"
    while [ ! -s rid.txt ] && [ $cnt -lt 120 ]; do
        sleep 1
        grep -a "RID 1 ReceiveFinishRealtime END" results/servers/server-* > rid.txt || true
        cnt=$(($cnt + 1))
        echo -n "."
    done

    # Start a user discovery bot server
    echo "STARTING UDB..."
    UDBCMD="../bin/udb --logLevel $DEBUGLEVEL --protoUserPath	udbProto.json --config udb.yaml -l 1"
    $UDBCMD >> $UDBOUT 2>&1 &
    PIDVAL=$!
    echo $PIDVAL >> results/serverpids
    echo "$UDBCMD -- $PIDVAL"
    rm rid.txt || true
    while [ ! -s rid.txt ] && [ $cnt -lt 30 ]; do
        sleep 1
        grep -a "Sending Poll message" results/udb-console.txt > rid.txt || true
        cnt=$(($cnt + 1))
        echo -n "."
    done

    echo "localhost:8440" > results/startgwserver.txt

    echo "DONE LETS DO STUFF"

else
    echo "Connecting to network defined at $NETWORKENTRYPOINT"
    echo $NETWORKENTRYPOINT > results/startgwserver.txt
fi

echo "DOWNLOADING TLS Cert..."
CMD="openssl s_client -showcerts -connect $(cat results/startgwserver.txt)"
echo $CMD
eval $CMD < /dev/null 2>&1 > "results/startgwcert.bin"
CMD="cat results/startgwcert.bin | openssl x509 -outform PEM"
echo $CMD
eval $CMD > "results/startgwcert.pem"
head "results/startgwcert.pem"

echo "DOWNLOADING NDF..."
CLIENTCMD="../bin/client getndf --gwhost $(cat results/startgwserver.txt) --cert results/startgwcert.pem"
eval $CLIENTCMD >> results/ndf.json 2>&1 &
PIDVAL=$!
echo "$CLIENTCMD -- $PIDVAL"
wait $PIDVAL

cat results/ndf.json | jq . | head -5

file results/ndf.json

if [ ! -s results/ndf.json ]
then
    echo "results/ndf.json is empty, cannot proceed"
    exit -1
fi