Skip to content
Snippets Groups Projects
Commit 86c5ca7d authored by Jono Wenger's avatar Jono Wenger
Browse files

Remote sync server

parent 1bd29011
No related branches found
No related tags found
1 merge request!67Remote sync server
......@@ -133,7 +133,6 @@ backup:
- results/
- bin/
dm:
stage: tests
image: $DOCKER_IMAGE
......@@ -149,7 +148,6 @@ dm:
- results/
- bin/
historical:
stage: tests
image: $DOCKER_IMAGE
......@@ -235,7 +233,6 @@ connect:
- results/
- bin/
broadcast:
stage: tests
image: $DOCKER_IMAGE
......@@ -251,7 +248,6 @@ broadcast:
- results/
- bin/
singleUse:
stage: tests
image: $DOCKER_IMAGE
......@@ -267,7 +263,6 @@ singleUse:
- results/
- bin/
ephemeralRegistration:
stage: tests
image: $DOCKER_IMAGE
......@@ -283,9 +278,7 @@ ephemeralRegistration:
- results/
- bin/
ud: #modify with your package name
ud:
stage: tests
image: $DOCKER_IMAGE
script:
......@@ -301,8 +294,7 @@ ud: #modify with your package name
- bin/
groupChat: #modify with your package name
groupChat:
stage: tests
image: $DOCKER_IMAGE
script:
......@@ -317,7 +309,20 @@ groupChat: #modify with your package name
- results/
- bin/
remoteSync:
stage: tests
image: $DOCKER_IMAGE
script:
- mkdir -p ~/.elixxir
- echo $PWD
- rm -fr results
- ./run.sh --run remoteSync
artifacts:
when: always
expire_in: '1 day'
paths:
- results/
- bin/
basice2e_protonet:
stage: tests
......@@ -341,7 +346,6 @@ basice2e_protonet:
# ONCE DEPLOYMENT IS COMPLETE.
- master
# basice2e_betanet:
# stage: tests
# image: $DOCKER_IMAGE
......@@ -375,7 +379,6 @@ smokeinfra:
- smokeinfra/results/
- bin/
smokeinfraBatch4:
stage: tests
image: $DOCKER_IMAGE
......@@ -427,7 +430,6 @@ rekey:
# - smokeinfra/results/
# - bin/
benchmark:
stage: benchmark
image: $DOCKER_IMAGE
......
......@@ -104,6 +104,7 @@ for BRANCH in $(echo "forcedbranch" $FBRANCH $FBRANCH2 $DEFAULTBRANCH); do
GPULIB2_URL=${GPULIB2_URL:="${REPOS_API}server/$BRANCH_URL/libpow.fatbin?job=build"}
CLIENT_REG_URL=${CLIENT_REG_URL:="${REPOS_API}client-registrar/$BRANCH_URL/registration$BIN"}
XXDK_WASM_URL=${XXDK_WASM_URL:="${REPOS_API}xxdk-wasm/$BRANCH_URL/xxdk.wasm?job=build"}
REMOTE_SYNC_SERVER_URL=${REMOTE_SYNC_SERVER_URL:="${REPOS_API}remoteSyncServer/$BRANCH_URL/remoteSyncServer$BIN"}
else
UDB_URL=${UDB_URL:="${REPOS_API}/$BRANCH/udb$BIN"}
SERVER_URL=${SERVER_URL:="${REPOS_API}/$BRANCH/server$BIN"}
......@@ -111,6 +112,7 @@ for BRANCH in $(echo "forcedbranch" $FBRANCH $FBRANCH2 $DEFAULTBRANCH); do
PERMISSIONING_URL=${PERMISSIONING_URL:="${REPOS_API}/$BRANCH/registration.stateless$BIN"}
CLIENT_URL=${CLIENT_URL:="${REPOS_API}/$BRANCH/client$BIN"}
XXDK_WASM_URL=${XXDK_WASM_URL:="${REPOS_API}/$BRANCH/xxdk.wasm?job=build"}
REMOTE_SYNC_SERVER_URL=${REMOTE_SYNC_SERVER_URL:="${REPOS_API}/$BRANCH/remoteSyncServer$BIN"}
fi
set -x
......@@ -145,11 +147,16 @@ for BRANCH in $(echo "forcedbranch" $FBRANCH $FBRANCH2 $DEFAULTBRANCH); do
curl -s -f -L -H "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" -o "$download_path/client-registrar" ${CLIENT_REG_URL}
fi
# Silently download the client registrar binary to the provisioning directory
# Silently download the xxdk WASM binary to the provisioning directory
if [ ! -f $download_path/xxdk.wasm ] && [[ "$XXDK_WASM_URL" != *"forcedbranch"* ]]; then
curl -s -f -L -H "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" -o "$download_path/xxdk.wasm" ${XXDK_WASM_URL}
fi
# Silently download the Haven remote sync server binary to the provisioning directory
if [ ! -f $download_path/remoteSyncServer ] && [[ "$REMOTE_SYNC_SERVER_URL" != *"forcedbranch"* ]]; then
curl -s -f -L -H "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" -o "$download_path/remoteSyncServer" ${REMOTE_SYNC_SERVER_URL}
fi
if [[ $2 == "d" ]]; then
# Silently download the Server binary to the provisioning directory
if [ ! -f $download_path/server-cuda ] && [[ "$SERVER_GPU_URL" != *"forcedbranch"* ]]; then
......@@ -185,6 +192,7 @@ fi
unset GPULIB2_URL
unset CLIENT_REG_URL
unset XXDK_WASM_URL
unset REMOTE_SYNC_SERVER_URL
done
# Make binaries executable
......
-----BEGIN CERTIFICATE-----
MIIFwTCCA6mgAwIBAgIUVjj8+S0ohr0Gjvl0vvLqbHGAA/IwDQYJKoZIhvcNAQEL
BQAwgYwxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTESMBAGA1UEBwwJQ2xhcmVt
b250MRAwDgYDVQQKDAdFbGl4eGlyMRQwEgYDVQQLDAtEZXZlbG9wbWVudDETMBEG
A1UEAwwKZWxpeHhpci5pbzEfMB0GCSqGSIb3DQEJARYQYWRtaW5AZWxpeHhpci5p
bzAeFw0yMzA3MTcyMjAxMjFaFw0zMzA3MTQyMjAxMjFaMIGMMQswCQYDVQQGEwJV
UzELMAkGA1UECAwCQ0ExEjAQBgNVBAcMCUNsYXJlbW9udDEQMA4GA1UECgwHRWxp
eHhpcjEUMBIGA1UECwwLRGV2ZWxvcG1lbnQxEzARBgNVBAMMCmVsaXh4aXIuaW8x
HzAdBgkqhkiG9w0BCQEWEGFkbWluQGVsaXh4aXIuaW8wggIiMA0GCSqGSIb3DQEB
AQUAA4ICDwAwggIKAoICAQC+oLMEXMExvdd5U0IfsiU0WPZmt3OvNq0QgXywuG2a
wn+t7tleTw/+5VAAWql6JVjbbDyZNj9oHQKh70u8wHVCit6xhqRNHBlevJi5/mwe
/+l3V21KCg03E67qLiqP3FQFCG7RLHl3KknGMKMGQQGAYfS865RJsCGQAwSs5O10
oFdac4Lu4MhmhzI2jtS+x14OPaXoAAyCb4PehtWd22kiJGsK98HSsCEHlT82aBD3
37TacIoCryhxPpQx3l2NTqYSD9sIovrV5H2rY6XV1DFc7KlE/aBp+Bf5YIWuwCgv
J4Mk97XkGQXxUCmvp5daMt1rmALx9BIwDUAxr2BUukZRzgSvmJkeO4RVhJrbAogB
WfnY7uyF3/dSrIluURuatyTMXNYaxjDNQ0f0LB6P0rruC6SsOH8q5F5GvI1He2vr
vgJyidbf1iYxbLw7uWVJpCI3S9E/Fw1+sGKHk+Kel9vNqlql3HcHk1srdHksLX66
70hRuWQ1SQXPViWxSNtGcD+/43aXiT8RFn/P0obX41EyoQE2fCliA9TTQner33mu
6Uf4D/3K298TEFetjG/8zvuOzbY6wPByAhX7QP4s2w8/G/jZ4JawX1ajXA9vNTTk
WiDOwHTPE6AlyBscYfi7M9miZgw1HI/XPAIt7YTMXkEsQ4S6q+xOzSyvPN7mJc9G
ywIDAQABoxkwFzAVBgNVHREEDjAMggplbGl4eGlyLmlvMA0GCSqGSIb3DQEBCwUA
A4ICAQAMtuCqVx5gzHGVyArIsfr/VN4bXh17L6mN5qR9w9CddhdKF4BZtdd0rvjq
ARbPkRHv1zK/Q0ORH6+n8fzFbErlAFnOuwJ1Z0w2tQ12svWme+A3/pLmTCQb2HOl
FTpbvbXefFMbd3rxz7v4mhuC8XqG52Wyu8yrP1d9pqMawkCGBUAZtOgoy52BOSRg
fiUaYaLcSPME3abTZjR/Y0eqCIe1yzpP+3bv6WKlrmuaruLcmrdNjTtp8uujzhp/
Dw/03DoPQVjxRJG5Du00lI63ue/PstjP7HbPySOjnbf6uNxajWUydsfSQSFoTuTv
jm7bgwCTQ6pzbb7fL73zZQrnpQxwDuaDfsYHFuofubU7IXRCVfyG5MMDM+7OCJrw
HemCF4tsOjCbUUcGhdh2VjzSL+evikkPbFdrqrwqjajH8WtPn78o8i7pWraSplMa
fEPmGS61eXnErfbM2rKdO4L9e+ksYIvuTqfnnaMxe2LnRuDc7YahWTnAXXzSjNkI
MsMl7hrI487dLFwN77NRz4IlULfKuCDKW3PSW7oY0xFpztT0aKxES+p8mwLB7tzV
/45gToeBBHKXVZwec0Ww8cJvxge35n8YPaOBm/gQruXHG1kkhxxzij1FIOv3+j85
SeNZQP8sjXr5w1SLoMxJm2QFycbmp8I/QC0RqGKINnvGNPIYQQ==
-----END CERTIFICATE-----
-----BEGIN PRIVATE KEY-----
MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQC+oLMEXMExvdd5
U0IfsiU0WPZmt3OvNq0QgXywuG2awn+t7tleTw/+5VAAWql6JVjbbDyZNj9oHQKh
70u8wHVCit6xhqRNHBlevJi5/mwe/+l3V21KCg03E67qLiqP3FQFCG7RLHl3KknG
MKMGQQGAYfS865RJsCGQAwSs5O10oFdac4Lu4MhmhzI2jtS+x14OPaXoAAyCb4Pe
htWd22kiJGsK98HSsCEHlT82aBD337TacIoCryhxPpQx3l2NTqYSD9sIovrV5H2r
Y6XV1DFc7KlE/aBp+Bf5YIWuwCgvJ4Mk97XkGQXxUCmvp5daMt1rmALx9BIwDUAx
r2BUukZRzgSvmJkeO4RVhJrbAogBWfnY7uyF3/dSrIluURuatyTMXNYaxjDNQ0f0
LB6P0rruC6SsOH8q5F5GvI1He2vrvgJyidbf1iYxbLw7uWVJpCI3S9E/Fw1+sGKH
k+Kel9vNqlql3HcHk1srdHksLX6670hRuWQ1SQXPViWxSNtGcD+/43aXiT8RFn/P
0obX41EyoQE2fCliA9TTQner33mu6Uf4D/3K298TEFetjG/8zvuOzbY6wPByAhX7
QP4s2w8/G/jZ4JawX1ajXA9vNTTkWiDOwHTPE6AlyBscYfi7M9miZgw1HI/XPAIt
7YTMXkEsQ4S6q+xOzSyvPN7mJc9GywIDAQABAoICAG6GYgUPenxKo9oWaepSsbuO
MR8REqPuQZ2mVhNoezQVvU2thJ5nnNCfSClMZ6MIcGKYePIkP9Mvrs0RH4qK0/IG
7mNju3Nx7Uu+cZa1ax72Jfkp2ZXqCgt7ABmLNuY+SwC4CPiCntJ21Fty67K+kg5L
vUikFlA7LNqAxX8xJHRQWxV4j9vMvKSlknmLitIqaeA74iuHi7k8yuqEpuplwqkn
EGFFkT2FHUF2STZUVDJDpGpagcpYoEw4ilf3o6pAhY1HYCm7xU+qsSpkFkzKQOyt
0t7LZuzKCaxvSFMv2pysfh86+F0OHSHy5piPxPNpNnQhqON8LedqZ2PnWSUG86mO
TsnnCY4EgreHk/g9S72ai1cmZlcQgF0/BrVpJG+PjeNd1duuSLrcsgeJLKFqkafq
rVluCOxNRW75DDyQYKU2vzvtPvvVlTeHlPRZnYI4UOJBuwzqpj9Q6TQnlIPXBvlo
MqfNWhlEWz+0ZYSC/gOQXMSYK1ba7sknsiFkLtbQNvjszS4ifSSSrI6OmKvmBmmk
easIV5D5UPfpDrCtg3DL1IJQ/jPNVyyvstW6KGZ4XhB8CPv1g+tlbXU3i6xHEzDI
ylJrtIszr42orPi2fYDFkMZzfVKs3wkyrIetWiPT1Jqhp/JuefNyYipO0jmTs+pt
wL8K7esY897BtqNcdRMRAoIBAQDjsnOJH6zskRs5X+Zy5iaj3TBDqert7IyhXRqG
BY4Rv7wGRhFGQNK45DOoq7vmHOlS9xGSI8+lK1AXC/kjDMqVZhvjK3s6lMnbKlKo
v1H/4VNQRfxvXRoJNm1ZHKQUXWftuNkxhBRzXXidJwcCr87BZedyyGzEpBZwmE3k
ILVo9L2ACZx5plYDUHoICswm1HiQdD/jh/CeZZVGtYPHNqIHjv8jRn47D8hyyaTW
C0cRbV/xXwk7/Yubw9mN5N+m3u87vqZwVAJ4bqq+KnFR4SKQkGpIWBIZGS4MPTpi
YN4OrAoXfqMTPzJZLwe9os1/Vp2N3YjSBD33rAMGVwf14W5FAoIBAQDWUqoUfZJ3
67jiQTH4BLyidb15pqEY/Kd1/CdGYLu0mPzo8LEzex76reL1mjyICYONugpNhI8R
bagBtzg+OR762UBttQUY7TmdlfrqLgveLCZMpqm1j1BTaXUpuUav/oJO4fbsZRLa
pfdmtQcTUaJIewvvbf2JejtI/4XDuMEVpQmct9aPnyag9+Sm9TDXDn67HTkzN+Og
TfcwVNqn87l3d9mBeEY0Nog26P10ZRwkAtavALpxWCraoEPq9SoZGGVarNiswQXH
yeaWJHlG0hWat3RhcyluxdNGK3PYlglvjckYizLIdc6WR4XZPSWtcRJKDVorNFuL
K93ozRCujvnPAoIBAFw6wlW+wNqqBC7/S2crFme1hoFB6GkrMeOCxOWYhm1mDpEz
/HdKdpBxzwL5XQjemRwD+9gILWmn268aKHYsDjrpu04PJMEb14+w2v6DUIGnEYoo
bxHk0lD8/NnlogiltR0zjLZj26zHJb3yOwK5l7P6xP9Xc8dK99PvvU00eobi64db
XUhhOeHpzSRIy1zHH3bTv7U4MRzX5NlZWOFunb8WCv6M5UCjKfxk9SHGO6kxdnVc
m27zSoNzl0HEfVsoXhsDJIPHY6DIotyYcIJMASYxePuIwPvtmRtYCbgjqI6k3Qmb
QA7hwFEjna0QrtBvky4NmAgJs9gIkskUo6LTB0kCggEAf8CsoyQq35LCFRPZjkZ8
jrZyKAZ8NqG7zGdundPhb4hWESAP8GjukWx/QfkbJdfN8/g2VfizMydTIxqsyuJP
piVD82EY1b/qPRX+iALEjj1w4vXhaVkw/1d4QFOxYAzsbcrh3g1tjKmrbCxyOEDL
5Dp8Zc8BdyZGGoA9ouoJJjbHIQSwP9IVO5QoVqEG6No6Dp0f0Hj7FT4EH2olIEH2
htEI9NTVvB/ZWCnPkscGXCzY/V4IxLFSchlgArj6zBSvrbokOe2g3XNrIIpDRETM
HfpGxCPaSa5f/o6+MhdVLoqsmRW4Qqd+ajPeBB0fzV/+D+gfnGtMOgv+7dr2m6ac
HwKCAQB/nEJNBbBOQvwGvt+7R9ZATx4p9yRcVNYXvAQr8KVN177Q1mo60ePG8G9B
CmKtorSLvog6fuZKLkIXOPc7ZUVmwY88ZEkK8xY05g0QBpcT1xA1jrqaX29ziZPF
P7jiMjHBR0Q/Vq72ZFVxtVEByRwl4EJ/4g8YEA88N4V0snXXH1Osg5OpUMBlZRCD
pDrL2HQ+JDFZvNQPvLrwNDNRtLvIHmySC7RZvjdprOfXBYc5fe+EA1uX19WKv5q9
6RWfKCa3g72VB/lZpiM/sn8rZvYX9fOWElauKSQv/BsE6egX6Au1H34CtO0PKyo3
xQ2pG7LwHHfRUWQW76QqD7AVusVD
-----END PRIVATE KEY-----
......@@ -27,6 +27,7 @@ echo "STARTING SERVERS..."
SERVERLOGS=$NETRESULTS/servers
GATEWAYLOGS=$NETRESULTS/gateways
UDBOUT=$NETRESULTS/udb-console.txt
RSSOUT=$NETRESULTS/remoteSyncServer-console.txt
mkdir -p $SERVERLOGS
mkdir -p $GATEWAYLOGS
......@@ -99,3 +100,10 @@ done
echo "localhost:1060" > $RESULTS/startgwserver.txt
# Start remote sync server
echo "STARTING REMOTE SYNC SERVER..."
RSSCMD="$NETBIN/remoteSyncServer --logLevel $DEBUGLEVEL --config network/remoteSyncServer.yaml"
$RSSCMD >> $RSSOUT 2>&1 &
PIDVAL=$!
echo $PIDVAL >> $NETRESULTS/serverpids
echo "$RSSCMD -- $PIDVAL"
##
# Remote Sync Server Configuration File
##
# Path where log file will be saved.
logPath: "results/network/remoteSyncServer.log"
# Port for server to listen on. It must be the only listener on this port.
port: 22841
# Path to CA-signed certificate files in PEM format.
signedCertPath: "keys/remoteSyncServer.crt"
signedKeyPath: "keys/remoteSyncServer.key"
# Duration that logged-in sessions are valid.
tokenTTL: 24h
# Path to CSV containing list of authorized users in "<username>,<password>" format.
credentialsCsvPath: "network/remoteSyncServerCredentials.csv"
# Base directory for synced files.
storageDir: "results/network/remoteSyncServer"
\ No newline at end of file
waldo,hunter2
\ No newline at end of file
......@@ -114,7 +114,7 @@ echo "Test report $timestamp" >> $testreport
if [ -z $run ]
then
TESTS=("basice2e" "backup" "dm" "historical" "channels" "fileTransfer" "connect" "broadcast" "groupChat" "ephemeralRegistration" "rekey" "singleUse" "channelsFileTransfer")
TESTS=("basice2e" "backup" "dm" "historical" "channels" "fileTransfer" "connect" "broadcast" "groupChat" "ephemeralRegistration" "rekey" "singleUse" "channelsFileTransfer" "remoteSync")
LOCALTESTS=("basice2e_local" "ud")
else
TESTS=(${run//,/ })
......
End Value for "someKey": {Version:0 Data:"someValueHere"}
Prefix Set: synchronized\
Set object to key "someKey": {Version:0 Data:"someValueHere"}
End Value for "someKey": {Version:0 Data:"someValueHere"}
Prefix Set: synchronized\
Update received for "someKey" (Created): nil -> {Version:0 Data:"someValueHere"}
# NOTE: This is verbose on purpose.
################################################################################
## Initial Set Up & Clean Up of Past Runs
################################################################################
set -e
if [ $# -gt 3 ]
then
echo "usage: $0 results_dir golds_dir"
exit
fi
REMOTESYNCRESULTS=$1
GOLDOUTPUT=$2
NDF=$3
DEBUGLEVEL=${DEBUGLEVEL-1}
CLIENTOUT=$REMOTESYNCRESULTS/clients
CLIENTCLEAN=$REMOTESYNCRESULTS/clients-cleaned
mkdir -p $CLIENTOUT
mkdir -p $CLIENTCLEAN
CLIENTOPTS="--password hello --ndf $NDF --verify-sends --sendDelay 100 --waitTimeout 360 -v $DEBUGLEVEL"
SERVEROPTS="--remoteSyncServerAddress 0.0.0.0:22841 --remoteCertPath keys/remoteSyncServer.crt"
#export GRPC_GO_LOG_VERBOSITY_LEVEL=99
#export GRPC_GO_LOG_SEVERITY_LEVEL=info
###############################################################################
# New Test Goes Here
###############################################################################
echo "TESTING REMOTE SYNCHRONISATION..."
CLIENTCMD="timeout 240s bin/client remoteSync $CLIENTOPTS -l $CLIENTOUT/client700a.log -s blobs/700a --remoteUsername waldo --remotePassword hunter2 $SERVEROPTS --remoteKey synchronized/someKey --remoteValue someValueHere"
eval $CLIENTCMD >> $CLIENTOUT/client700a.txt &
PIDVAL=$!
echo "$CLIENTCMD -- $PIDVAL"
wait $PIDVAL
CLIENTCMD="timeout 240s bin/client remoteSync $CLIENTOPTS -l $CLIENTOUT/client700b.log -s blobs/700b --remoteUsername waldo --remotePassword hunter2 $SERVEROPTS --remoteKey synchronized/someKey"
eval $CLIENTCMD >> $CLIENTOUT/client700b.txt &
PIDVAL=$!
echo "$CLIENTCMD -- $PIDVAL"
wait $PIDVAL
echo "TESTS EXITED SUCCESSFULLY, CHECKING OUTPUT..."
cp $CLIENTOUT/*.txt $CLIENTCLEAN/
sed -i.bak 's/Sending\ to\ .*\:/Sent:/g' $CLIENTCLEAN/client*.txt
sed -i.bak 's/Message\ from\ .*, .* Received:/Received:/g' $CLIENTCLEAN/client*.txt
sed -i.bak 's/ERROR.*Signature/Signature/g' $CLIENTCLEAN/client*.txt
sed -i.bak 's/[Aa]uthenticat.*$//g' $CLIENTCLEAN/client*.txt
rm $CLIENTCLEAN/client*.txt.bak
for C in $(ls -1 $CLIENTCLEAN | grep -v client11[01]); do
sort -o tmp $CLIENTCLEAN/$C || true
cp tmp $CLIENTCLEAN/$C
# uniq tmp $CLIENTCLEAN/$C || true
done
set -e
set +x
diff -aru $GOLDOUTPUT $CLIENTCLEAN
echo "NO OUTPUT ERRORS, SUCCESS!"
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment