diff --git a/fileTransfer2/batchBuilder.go b/fileTransfer2/batchBuilder.go
new file mode 100644
index 0000000000000000000000000000000000000000..2573ffb5e98f3af814547e0c5c735d68009cdbe5
--- /dev/null
+++ b/fileTransfer2/batchBuilder.go
@@ -0,0 +1,82 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileTransfer2
+
+import (
+ "encoding/binary"
+ jww "github.com/spf13/jwalterweatherman"
+ "gitlab.com/elixxir/client/fileTransfer2/store"
+ "gitlab.com/elixxir/client/stoppable"
+ "gitlab.com/elixxir/crypto/fastRNG"
+ "gitlab.com/xx_network/crypto/csprng"
+ "go.uber.org/ratelimit"
+ "time"
+)
+
+const (
+ // Duration to wait before adding a partially filled part packet to the send
+ // channel.
+ unfilledPacketTimeout = 100 * time.Millisecond
+)
+
+// batchBuilderThread creates batches of file parts as they become available and
+// buffer them to send. Also rate limits adding to the buffer.
+func (m *manager) batchBuilderThread(stop *stoppable.Single) {
+ // Calculate the average amount of data sent via SendManyCMIX
+ avgNumMessages := (minPartsSendPerRound + maxPartsSendPerRound) / 2
+ avgSendSize := avgNumMessages * 8192
+
+ // Calculate rate and make rate limiter
+ rate := m.params.MaxThroughput / avgSendSize
+ rl := ratelimit.New(rate, ratelimit.WithoutSlack)
+
+ for {
+ numParts := generateRandomPacketSize(m.rng)
+ packet := make([]store.Part, 0, numParts)
+ delayedTimer := NewDelayedTimer(unfilledPacketTimeout)
+ loop:
+ for cap(packet) > len(packet) {
+ select {
+ case <-stop.Quit():
+ delayedTimer.Stop()
+ jww.DEBUG.Printf("[FT] Stopping file part packing thread " +
+ "while packing: stoppable triggered.")
+ stop.ToStopped()
+ return
+ case <-*delayedTimer.C:
+ break loop
+ case p := <-m.batchQueue:
+ packet = append(packet, p)
+ delayedTimer.Start()
+ }
+ }
+
+ // Rate limiter
+ rl.Take()
+ m.sendQueue <- packet
+ }
+}
+
+// generateRandomPacketSize returns a random number between minPartsSendPerRound
+// and maxPartsSendPerRound, inclusive.
+func generateRandomPacketSize(rngGen *fastRNG.StreamGenerator) int {
+ rng := rngGen.GetStream()
+ defer rng.Close()
+
+ // Generate random bytes
+ b, err := csprng.Generate(8, rng)
+ if err != nil {
+ jww.FATAL.Panicf(getRandomNumPartsRandPanic, err)
+ }
+
+ // Convert bytes to integer
+ num := binary.LittleEndian.Uint64(b)
+
+ // Return random number that is minPartsSendPerRound <= num <= max
+ return int((num % (maxPartsSendPerRound)) + minPartsSendPerRound)
+}
diff --git a/fileTransfer2/callbackTracker/callbackTracker.go b/fileTransfer2/callbackTracker/callbackTracker.go
new file mode 100644
index 0000000000000000000000000000000000000000..6cf7a641c10b2fccf96b46097a5c18dbabea3655
--- /dev/null
+++ b/fileTransfer2/callbackTracker/callbackTracker.go
@@ -0,0 +1,99 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package callbackTracker
+
+import (
+ "gitlab.com/elixxir/client/stoppable"
+ "gitlab.com/xx_network/primitives/netTime"
+ "sync"
+ "time"
+)
+
+type callback func(err error)
+
+// callbackTracker tracks the fileTransfer.SentProgressCallback and
+// information on when to call it. The callback will be called on each send,
+// unless the time since the lastCall is smaller than the period. In that case,
+// a callback is marked as scheduled and waits to be called at the end of the
+// period. A callback is called once every period, regardless of the number of
+// sends that occur.
+type callbackTracker struct {
+ period time.Duration // How often to call the callback
+ lastCall time.Time // Timestamp of the last call
+ scheduled bool // Denotes if callback call is scheduled
+ complete bool // Denotes if the callback should not be called
+ stop *stoppable.Single // Stops the scheduled callback from triggering
+ cb callback
+ mux sync.RWMutex
+}
+
+// newCallbackTracker creates a new and unused sentCallbackTracker.
+func newCallbackTracker(
+ cb callback, period time.Duration, stop *stoppable.Single) *callbackTracker {
+ return &callbackTracker{
+ period: period,
+ lastCall: time.Time{},
+ scheduled: false,
+ complete: false,
+ stop: stop,
+ cb: cb,
+ }
+}
+
+// call triggers the progress callback with the most recent progress from the
+// sentProgressTracker. If a callback has been called within the last period,
+// then a new call is scheduled to occur at the beginning of the next period. If
+// a call is already scheduled, then nothing happens; when the callback is
+// finally called, it will do so with the most recent changes.
+func (ct *callbackTracker) call(err error) {
+ ct.mux.RLock()
+ // Exit if a callback is already scheduled
+ if (ct.scheduled || ct.complete) && err == nil {
+ ct.mux.RUnlock()
+ return
+ }
+
+ ct.mux.RUnlock()
+ ct.mux.Lock()
+ defer ct.mux.Unlock()
+
+ if (ct.scheduled || ct.complete) && err == nil {
+ return
+ }
+
+ // Mark callback complete if an error is passed
+ ct.complete = err != nil
+
+ // Check if a callback has occurred within the last period
+ timeSinceLastCall := netTime.Since(ct.lastCall)
+ if timeSinceLastCall > ct.period {
+
+ // If no callback occurred, then trigger the callback now
+ go ct.cb(err)
+ ct.lastCall = netTime.Now()
+ } else {
+ // If a callback did occur, then schedule a new callback to occur at the
+ // start of the next period
+ ct.scheduled = true
+ go func() {
+ timer := time.NewTimer(ct.period - timeSinceLastCall)
+ select {
+ case <-ct.stop.Quit():
+ timer.Stop()
+ ct.stop.ToStopped()
+ return
+ case <-timer.C:
+ ct.mux.Lock()
+ go ct.cb(err)
+ ct.lastCall = netTime.Now()
+ ct.scheduled = false
+ ct.mux.Unlock()
+ }
+ }()
+ }
+}
diff --git a/fileTransfer2/callbackTracker/callbackTracker_test.go b/fileTransfer2/callbackTracker/callbackTracker_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..f14d368ee0fb6fbaa0ef9bdd1c2ac4b17c511755
--- /dev/null
+++ b/fileTransfer2/callbackTracker/callbackTracker_test.go
@@ -0,0 +1,148 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package callbackTracker
+
+import (
+ "github.com/pkg/errors"
+ "gitlab.com/elixxir/client/stoppable"
+ "reflect"
+ "testing"
+ "time"
+)
+
+// Tests that newCallbackTracker returns a new callbackTracker with all the
+// expected values.
+func Test_newCallbackTracker(t *testing.T) {
+ expected := &callbackTracker{
+ period: time.Millisecond,
+ lastCall: time.Time{},
+ scheduled: false,
+ complete: false,
+ stop: stoppable.NewSingle("Test_newCallbackTracker"),
+ }
+
+ newCT := newCallbackTracker(nil, expected.period, expected.stop)
+ newCT.cb = nil
+
+ if !reflect.DeepEqual(expected, newCT) {
+ t.Errorf("New callbackTracker does not match expected."+
+ "\nexpected: %+v\nreceived: %+v", expected, newCT)
+ }
+}
+
+// Tests four test cases of callbackTracker.call:
+// 1. An initial call is not scheduled.
+// 2. A second call within the periods is only called after the period.
+// 3. An error sets the callback to complete.
+// 4. No more callbacks will be called after set to complete.
+func Test_callbackTracker_call(t *testing.T) {
+ cbChan := make(chan error, 10)
+ cb := func(err error) { cbChan <- err }
+ stop := stoppable.NewSingle("Test_callbackTracker_call")
+ ct := newCallbackTracker(cb, 250*time.Millisecond, stop)
+
+ // Test that the initial call is unscheduled and is called before the period
+ go ct.call(nil)
+
+ select {
+ case r := <-cbChan:
+ if r != nil {
+ t.Errorf("Received error: %+v", r)
+ }
+ case <-time.After(25 * time.Millisecond):
+ t.Error("Timed out waiting for callback.")
+ }
+
+ // Test that another call within the period is called only after the period
+ // is reached
+ go ct.call(nil)
+
+ select {
+ case <-cbChan:
+ t.Error("Callback called too soon.")
+
+ case <-time.After(25 * time.Millisecond):
+ ct.mux.RLock()
+ if !ct.scheduled {
+ t.Error("Callback is not scheduled when it should be.")
+ }
+ ct.mux.RUnlock()
+ select {
+ case r := <-cbChan:
+ if r != nil {
+ t.Errorf("Received error: %+v", r)
+ }
+ case <-time.After(ct.period):
+ t.Errorf("Callback not called after period %s.", ct.period)
+
+ if ct.scheduled {
+ t.Error("Callback is scheduled when it should not be.")
+ }
+ }
+ }
+
+ // Test that calling with an error sets the callback to complete
+ expectedErr := errors.New("test error")
+ go ct.call(expectedErr)
+
+ select {
+ case r := <-cbChan:
+ if r != expectedErr {
+ t.Errorf("Received incorrect error.\nexpected: %v\nreceived: %v",
+ expectedErr, r)
+ }
+ if !ct.complete {
+ t.Error("Callback is not marked complete when it should be.")
+ }
+ case <-time.After(ct.period + 15*time.Millisecond):
+ t.Errorf("Callback not called after period %s.",
+ ct.period+15*time.Millisecond)
+ }
+
+ // Tests that all callback calls after an error are blocked
+ go ct.call(nil)
+
+ select {
+ case r := <-cbChan:
+ t.Errorf("Received callback when it should have been completed: %+v", r)
+ case <-time.After(ct.period):
+ }
+}
+
+// Tests that callbackTracker.call does not call on the callback when the
+// stoppable is triggered.
+func Test_callbackTracker_call_stop(t *testing.T) {
+ cbChan := make(chan error, 10)
+ cb := func(err error) { cbChan <- err }
+ stop := stoppable.NewSingle("Test_callbackTracker_call")
+ ct := newCallbackTracker(cb, 250*time.Millisecond, stop)
+
+ go ct.call(nil)
+
+ select {
+ case r := <-cbChan:
+ if r != nil {
+ t.Errorf("Received error: %+v", r)
+ }
+ case <-time.After(25 * time.Millisecond):
+ t.Error("Timed out waiting for callback.")
+ }
+
+ go ct.call(nil)
+
+ err := stop.Close()
+ if err != nil {
+ t.Errorf("Failed closing stoppable: %+v", err)
+ }
+
+ select {
+ case <-cbChan:
+ t.Error("Callback called.")
+ case <-time.After(ct.period * 2):
+ }
+}
diff --git a/fileTransfer2/callbackTracker/manager.go b/fileTransfer2/callbackTracker/manager.go
new file mode 100644
index 0000000000000000000000000000000000000000..e6c4cf437ad0d625454667950d2a136baa8f20d2
--- /dev/null
+++ b/fileTransfer2/callbackTracker/manager.go
@@ -0,0 +1,99 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package callbackTracker
+
+import (
+ jww "github.com/spf13/jwalterweatherman"
+ "gitlab.com/elixxir/client/stoppable"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "strconv"
+ "sync"
+ "time"
+)
+
+// Manager tracks the callbacks for each transfer.
+type Manager struct {
+ // Map of transfers and their list of callbacks
+ callbacks map[ftCrypto.TransferID][]*callbackTracker
+
+ // List of multi stoppables used to stop callback trackers; each multi
+ // stoppable contains a single stoppable for each callback.
+ stops map[ftCrypto.TransferID]*stoppable.Multi
+
+ mux sync.RWMutex
+}
+
+// NewManager initializes a new callback tracker Manager.
+func NewManager() *Manager {
+ m := &Manager{
+ callbacks: make(map[ftCrypto.TransferID][]*callbackTracker),
+ stops: make(map[ftCrypto.TransferID]*stoppable.Multi),
+ }
+
+ return m
+}
+
+// AddCallback adds a callback to the list of callbacks for the given transfer
+// ID and calls it regardless of the callback tracker status.
+func (m *Manager) AddCallback(
+ tid *ftCrypto.TransferID, cb callback, period time.Duration) {
+ m.mux.Lock()
+ defer m.mux.Unlock()
+
+ // Create new entries for this transfer ID if none exist
+ if _, exists := m.callbacks[*tid]; !exists {
+ m.callbacks[*tid] = []*callbackTracker{}
+ m.stops[*tid] = stoppable.NewMulti("FileTransfer/" + tid.String())
+ }
+
+ // Generate the stoppable and add it to the transfer's multi stoppable
+ stop := stoppable.NewSingle(makeStoppableName(tid, len(m.callbacks[*tid])))
+ m.stops[*tid].Add(stop)
+
+ // Create new callback tracker and add to the map
+ ct := newCallbackTracker(cb, period, stop)
+ m.callbacks[*tid] = append(m.callbacks[*tid], ct)
+
+ // Call the callback
+ go cb(nil)
+}
+
+// Call triggers each callback for the given transfer ID and passes along the
+// given error.
+func (m *Manager) Call(tid *ftCrypto.TransferID, err error) {
+ m.mux.Lock()
+ defer m.mux.Unlock()
+
+ for _, cb := range m.callbacks[*tid] {
+ go cb.call(err)
+ }
+}
+
+// Delete stops all scheduled stoppables for the given transfer and deletes the
+// callbacks from the map.
+func (m *Manager) Delete(tid *ftCrypto.TransferID) {
+ m.mux.Lock()
+ defer m.mux.Unlock()
+
+ // Stop the stoppable if the stoppable still exists
+ stop, exists := m.stops[*tid]
+ if exists {
+ if err := stop.Close(); err != nil {
+ jww.ERROR.Printf("[FT] Failed to stop progress callbacks: %+v", err)
+ }
+ }
+
+ // Delete callbacks and stoppables
+ delete(m.callbacks, *tid)
+ delete(m.stops, *tid)
+}
+
+// makeStoppableName generates a unique name for the callback stoppable.
+func makeStoppableName(tid *ftCrypto.TransferID, callbackNum int) string {
+ return tid.String() + "/" + strconv.Itoa(callbackNum)
+}
diff --git a/fileTransfer2/callbackTracker/manager_test.go b/fileTransfer2/callbackTracker/manager_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..4226807f18a728345fc4907758bd27ae21b6dcc1
--- /dev/null
+++ b/fileTransfer2/callbackTracker/manager_test.go
@@ -0,0 +1,176 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package callbackTracker
+
+import (
+ "errors"
+ "gitlab.com/elixxir/client/stoppable"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/xx_network/crypto/csprng"
+ "io"
+ "math/rand"
+ "reflect"
+ "sync"
+ "testing"
+ "time"
+)
+
+// Tests that NewManager returns the expected Manager.
+func TestNewManager(t *testing.T) {
+ expected := &Manager{
+ callbacks: make(map[ftCrypto.TransferID][]*callbackTracker),
+ stops: make(map[ftCrypto.TransferID]*stoppable.Multi),
+ }
+
+ newManager := NewManager()
+
+ if !reflect.DeepEqual(expected, newManager) {
+ t.Errorf("New Manager does not match expected."+
+ "\nexpected: %+v\nreceived: %+v", expected, newManager)
+ }
+}
+
+// Tests that Manager.AddCallback adds the callback to the list, creates and
+// adds a stoppable to the list, and that the callback is called.
+func TestManager_AddCallback(t *testing.T) {
+ m := NewManager()
+
+ cbChan := make(chan error, 10)
+ cb := func(err error) { cbChan <- err }
+ tid := &ftCrypto.TransferID{5}
+ m.AddCallback(tid, cb, 0)
+
+ // Check that the callback was called
+ select {
+ case <-cbChan:
+ case <-time.After(25 * time.Millisecond):
+ t.Error("Timed out waiting for callback to be called.")
+ }
+
+ // Check that the callback was added
+ if _, exists := m.callbacks[*tid]; !exists {
+ t.Errorf("No callback list found for transfer ID %s.", tid)
+ } else if len(m.callbacks[*tid]) != 1 {
+ t.Errorf("Incorrect number of callbacks.\nexpected: %d\nreceived: %d",
+ 1, len(m.callbacks[*tid]))
+ }
+
+ // Check that the stoppable was added
+ if _, exists := m.stops[*tid]; !exists {
+ t.Errorf("No stoppable list found for transfer ID %s.", tid)
+ }
+}
+
+// Tests that Manager.Call calls al the callbacks associated with the transfer
+// ID.
+func TestManager_Call(t *testing.T) {
+ m := NewManager()
+ tid := &ftCrypto.TransferID{5}
+ n := 10
+ cbChans := make([]chan error, n)
+ cbs := make([]func(err error), n)
+ for i := range cbChans {
+ cbChan := make(chan error, 10)
+ cbs[i] = func(err error) { cbChan <- err }
+ cbChans[i] = cbChan
+ }
+
+ // Add callbacks
+ for i := range cbs {
+ m.AddCallback(tid, cbs[i], 0)
+
+ // Receive channel from first call
+ select {
+ case <-cbChans[i]:
+ case <-time.After(25 * time.Millisecond):
+ t.Errorf("Callback #%d never called.", i)
+ }
+ }
+
+ // Call callbacks
+ m.Call(tid, errors.New("test"))
+
+ // Check to make sure callbacks were called
+ var wg sync.WaitGroup
+ for i := range cbs {
+ wg.Add(1)
+ go func(i int) {
+ select {
+ case r := <-cbChans[i]:
+ if r == nil {
+ t.Errorf("Callback #%d did not receive an error.", i)
+ }
+ case <-time.After(25 * time.Millisecond):
+ t.Errorf("Callback #%d never called.", i)
+ }
+
+ wg.Done()
+ }(i)
+ }
+
+ wg.Wait()
+}
+
+// Tests that Manager.Delete removes all callbacks and stoppables from the list.
+func TestManager_Delete(t *testing.T) {
+ m := NewManager()
+
+ cbChan := make(chan error, 10)
+ cb := func(err error) { cbChan <- err }
+ tid := &ftCrypto.TransferID{5}
+ m.AddCallback(tid, cb, 0)
+
+ m.Delete(tid)
+
+ // Check that the callback was deleted
+ if _, exists := m.callbacks[*tid]; exists {
+ t.Errorf("Callback list found for transfer ID %s.", tid)
+ }
+
+ // Check that the stoppable was deleted
+ if _, exists := m.stops[*tid]; exists {
+ t.Errorf("Stoppable list found for transfer ID %s.", tid)
+ }
+}
+
+// Consistency test of makeStoppableName.
+func Test_makeStoppableName_Consistency(t *testing.T) {
+ rng := NewPrng(42)
+ expectedValues := []string{
+ "U4x/lrFkvxuXu59LtHLon1sUhPJSCcnZND6SugndnVI=/0",
+ "39ebTXZCm2F6DJ+fDTulWwzA1hRMiIU1hBrL4HCbB1g=/1",
+ "CD9h03W8ArQd9PkZKeGP2p5vguVOdI6B555LvW/jTNw=/2",
+ "uoQ+6NY+jE/+HOvqVG2PrBPdGqwEzi6ih3xVec+ix44=/3",
+ "GwuvrogbgqdREIpC7TyQPKpDRlp4YgYWl4rtDOPGxPM=/4",
+ "rnvD4ElbVxL+/b4MECiH4QDazS2IX2kstgfaAKEcHHA=/5",
+ "ceeWotwtwlpbdLLhKXBeJz8FySMmgo4rBW44F2WOEGE=/6",
+ "SYlH/fNEQQ7UwRYCP6jjV2tv7Sf/iXS6wMr9mtBWkrE=/7",
+ "NhnnOJZN/ceejVNDc2Yc/WbXT+weG4lJGrcjbkt1IWI=/8",
+ "kM8r60LDyicyhWDxqsBnzqbov0bUqytGgEAsX7KCDog=/9",
+ }
+
+ for i, expected := range expectedValues {
+ tid, err := ftCrypto.NewTransferID(rng)
+ if err != nil {
+ t.Errorf("Failed to generated transfer ID #%d: %+v", i, err)
+ }
+
+ name := makeStoppableName(&tid, i)
+ if expected != name {
+ t.Errorf("Stoppable name does not match expected."+
+ "\nexpected: %q\nreceived: %q", expected, name)
+ }
+ }
+}
+
+// Prng is a PRNG that satisfies the csprng.Source interface.
+type Prng struct{ prng io.Reader }
+
+func NewPrng(seed int64) csprng.Source { return &Prng{rand.New(rand.NewSource(seed))} }
+func (s *Prng) Read(b []byte) (int, error) { return s.prng.Read(b) }
+func (s *Prng) SetSeed([]byte) error { return nil }
diff --git a/fileTransfer2/delayedTimer.go b/fileTransfer2/delayedTimer.go
new file mode 100644
index 0000000000000000000000000000000000000000..5f62cbc5c799623b9705a2395950eb5d954f2232
--- /dev/null
+++ b/fileTransfer2/delayedTimer.go
@@ -0,0 +1,56 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileTransfer2
+
+import "time"
+
+// The DelayedTimer type represents a single event manually started.
+// When the DelayedTimer expires, the current time will be sent on C.
+// A DelayedTimer must be created with NewDelayedTimer.
+type DelayedTimer struct {
+ d time.Duration
+ t *time.Timer
+ C *<-chan time.Time
+}
+
+// NewDelayedTimer creates a new DelayedTimer that will send the current time on
+// its channel after at least duration d once it is started.
+func NewDelayedTimer(d time.Duration) *DelayedTimer {
+ c := make(<-chan time.Time)
+ return &DelayedTimer{
+ d: d,
+ C: &c,
+ }
+}
+
+// Start starts the timer that will send the current time on its channel after
+// at least duration d. If it is already running or stopped, it does nothing.
+func (dt *DelayedTimer) Start() {
+ if dt.t == nil {
+ dt.t = time.NewTimer(dt.d)
+ dt.C = &dt.t.C
+ }
+}
+
+// Stop prevents the Timer from firing.
+// It returns true if the call stops the timer, false if the timer has already
+// expired, been stopped, or was never started.
+func (dt *DelayedTimer) Stop() bool {
+ if dt.t == nil {
+ return false
+ }
+
+ return dt.t.Stop()
+}
diff --git a/fileTransfer2/ftMessages.pb.go b/fileTransfer2/ftMessages.pb.go
new file mode 100644
index 0000000000000000000000000000000000000000..ea810773b08764691951920a1e92953f1711a54b
--- /dev/null
+++ b/fileTransfer2/ftMessages.pb.go
@@ -0,0 +1,227 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+///////////////////////////////////////////////////////////////////////////////
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// protoc-gen-go v1.26.0
+// protoc v3.17.3
+// source: fileTransfer/ftMessages.proto
+
+package fileTransfer2
+
+import (
+ protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+ protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+ reflect "reflect"
+ sync "sync"
+)
+
+const (
+ // Verify that this generated code is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+ // Verify that runtime/protoimpl is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// NewFileTransfer is transmitted first on the initialization of a file transfer
+// to inform the receiver about the incoming file.
+type NewFileTransfer struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ FileName string `protobuf:"bytes,1,opt,name=fileName,proto3" json:"fileName,omitempty"` // Name of the file; max 48 characters
+ FileType string `protobuf:"bytes,2,opt,name=fileType,proto3" json:"fileType,omitempty"` // Tag of file; max 8 characters
+ TransferKey []byte `protobuf:"bytes,3,opt,name=transferKey,proto3" json:"transferKey,omitempty"` // 256 bit encryption key to identify the transfer
+ TransferMac []byte `protobuf:"bytes,4,opt,name=transferMac,proto3" json:"transferMac,omitempty"` // 256 bit MAC of the entire file
+ NumParts uint32 `protobuf:"varint,5,opt,name=numParts,proto3" json:"numParts,omitempty"` // Number of file parts
+ Size uint32 `protobuf:"varint,6,opt,name=size,proto3" json:"size,omitempty"` // The size of the file; max of 4 mB
+ Retry float32 `protobuf:"fixed32,7,opt,name=retry,proto3" json:"retry,omitempty"` // Used to determine how many times to retry sending
+ Preview []byte `protobuf:"bytes,8,opt,name=preview,proto3" json:"preview,omitempty"` // A preview of the file; max of 4 kB
+}
+
+func (x *NewFileTransfer) Reset() {
+ *x = NewFileTransfer{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_fileTransfer_ftMessages_proto_msgTypes[0]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *NewFileTransfer) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*NewFileTransfer) ProtoMessage() {}
+
+func (x *NewFileTransfer) ProtoReflect() protoreflect.Message {
+ mi := &file_fileTransfer_ftMessages_proto_msgTypes[0]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use NewFileTransfer.ProtoReflect.Descriptor instead.
+func (*NewFileTransfer) Descriptor() ([]byte, []int) {
+ return file_fileTransfer_ftMessages_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *NewFileTransfer) GetFileName() string {
+ if x != nil {
+ return x.FileName
+ }
+ return ""
+}
+
+func (x *NewFileTransfer) GetFileType() string {
+ if x != nil {
+ return x.FileType
+ }
+ return ""
+}
+
+func (x *NewFileTransfer) GetTransferKey() []byte {
+ if x != nil {
+ return x.TransferKey
+ }
+ return nil
+}
+
+func (x *NewFileTransfer) GetTransferMac() []byte {
+ if x != nil {
+ return x.TransferMac
+ }
+ return nil
+}
+
+func (x *NewFileTransfer) GetNumParts() uint32 {
+ if x != nil {
+ return x.NumParts
+ }
+ return 0
+}
+
+func (x *NewFileTransfer) GetSize() uint32 {
+ if x != nil {
+ return x.Size
+ }
+ return 0
+}
+
+func (x *NewFileTransfer) GetRetry() float32 {
+ if x != nil {
+ return x.Retry
+ }
+ return 0
+}
+
+func (x *NewFileTransfer) GetPreview() []byte {
+ if x != nil {
+ return x.Preview
+ }
+ return nil
+}
+
+var File_fileTransfer_ftMessages_proto protoreflect.FileDescriptor
+
+var file_fileTransfer_ftMessages_proto_rawDesc = []byte{
+ 0x0a, 0x1d, 0x66, 0x69, 0x6c, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x2f, 0x66,
+ 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12,
+ 0x05, 0x70, 0x61, 0x72, 0x73, 0x65, 0x22, 0xed, 0x01, 0x0a, 0x0f, 0x4e, 0x65, 0x77, 0x46, 0x69,
+ 0x6c, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69,
+ 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69,
+ 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x54, 0x79,
+ 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x54, 0x79,
+ 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x4b, 0x65,
+ 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65,
+ 0x72, 0x4b, 0x65, 0x79, 0x12, 0x20, 0x0a, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72,
+ 0x4d, 0x61, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73,
+ 0x66, 0x65, 0x72, 0x4d, 0x61, 0x63, 0x12, 0x1a, 0x0a, 0x08, 0x6e, 0x75, 0x6d, 0x50, 0x61, 0x72,
+ 0x74, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x6e, 0x75, 0x6d, 0x50, 0x61, 0x72,
+ 0x74, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d,
+ 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x74, 0x72, 0x79, 0x18,
+ 0x07, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x72, 0x65, 0x74, 0x72, 0x79, 0x12, 0x18, 0x0a, 0x07,
+ 0x70, 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70,
+ 0x72, 0x65, 0x76, 0x69, 0x65, 0x77, 0x42, 0x0f, 0x5a, 0x0d, 0x66, 0x69, 0x6c, 0x65, 0x54, 0x72,
+ 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x2f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+ file_fileTransfer_ftMessages_proto_rawDescOnce sync.Once
+ file_fileTransfer_ftMessages_proto_rawDescData = file_fileTransfer_ftMessages_proto_rawDesc
+)
+
+func file_fileTransfer_ftMessages_proto_rawDescGZIP() []byte {
+ file_fileTransfer_ftMessages_proto_rawDescOnce.Do(func() {
+ file_fileTransfer_ftMessages_proto_rawDescData = protoimpl.X.CompressGZIP(file_fileTransfer_ftMessages_proto_rawDescData)
+ })
+ return file_fileTransfer_ftMessages_proto_rawDescData
+}
+
+var file_fileTransfer_ftMessages_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_fileTransfer_ftMessages_proto_goTypes = []interface{}{
+ (*NewFileTransfer)(nil), // 0: parse.NewFileTransfer
+}
+var file_fileTransfer_ftMessages_proto_depIdxs = []int32{
+ 0, // [0:0] is the sub-list for method output_type
+ 0, // [0:0] is the sub-list for method input_type
+ 0, // [0:0] is the sub-list for extension type_name
+ 0, // [0:0] is the sub-list for extension extendee
+ 0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_fileTransfer_ftMessages_proto_init() }
+func file_fileTransfer_ftMessages_proto_init() {
+ if File_fileTransfer_ftMessages_proto != nil {
+ return
+ }
+ if !protoimpl.UnsafeEnabled {
+ file_fileTransfer_ftMessages_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*NewFileTransfer); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ }
+ type x struct{}
+ out := protoimpl.TypeBuilder{
+ File: protoimpl.DescBuilder{
+ GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+ RawDescriptor: file_fileTransfer_ftMessages_proto_rawDesc,
+ NumEnums: 0,
+ NumMessages: 1,
+ NumExtensions: 0,
+ NumServices: 0,
+ },
+ GoTypes: file_fileTransfer_ftMessages_proto_goTypes,
+ DependencyIndexes: file_fileTransfer_ftMessages_proto_depIdxs,
+ MessageInfos: file_fileTransfer_ftMessages_proto_msgTypes,
+ }.Build()
+ File_fileTransfer_ftMessages_proto = out.File
+ file_fileTransfer_ftMessages_proto_rawDesc = nil
+ file_fileTransfer_ftMessages_proto_goTypes = nil
+ file_fileTransfer_ftMessages_proto_depIdxs = nil
+}
diff --git a/fileTransfer2/ftMessages.proto b/fileTransfer2/ftMessages.proto
new file mode 100644
index 0000000000000000000000000000000000000000..2b1ae1832b4dc449589659ad2f9d3a1c3c6d79ca
--- /dev/null
+++ b/fileTransfer2/ftMessages.proto
@@ -0,0 +1,24 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+///////////////////////////////////////////////////////////////////////////////
+
+syntax = "proto3";
+
+package parse;
+option go_package = "fileTransfer/";
+
+// NewFileTransfer is transmitted first on the initialization of a file transfer
+// to inform the receiver about the incoming file.
+message NewFileTransfer {
+ string fileName = 1; // Name of the file
+ string fileType = 2; // String that indicates type of file
+ bytes transferKey = 3; // 256-bit encryption key
+ bytes transferMac = 4; // 256-bit MAC of the entire file
+ uint32 numParts = 5; // Number of file parts
+ uint32 size = 6; // The size of the file, in bytes
+ float retry = 7; // Determines how many times to retry sending
+ bytes preview = 8; // A preview of the file
+}
\ No newline at end of file
diff --git a/fileTransfer2/generateProto.sh b/fileTransfer2/generateProto.sh
new file mode 100644
index 0000000000000000000000000000000000000000..324062a3c85de33896a9fc4ebb3210a798cbe49f
--- /dev/null
+++ b/fileTransfer2/generateProto.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+#
+# Copyright © 2020 xx network SEZC ///
+# ///
+# Use of this source code is governed by a license that can be found in the ///
+# LICENSE file ///
+#
+
+protoc --go_out=paths=source_relative:. fileTransfer/ftMessages.proto
diff --git a/fileTransfer2/info.go b/fileTransfer2/info.go
new file mode 100644
index 0000000000000000000000000000000000000000..66bd12e635cc8e3f186b5f22b91792427f713bb3
--- /dev/null
+++ b/fileTransfer2/info.go
@@ -0,0 +1,33 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileTransfer2
+
+import (
+ "encoding/json"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+)
+
+type TransferInfo struct {
+ FileName string // Name of the file
+ FileType string // String that indicates type of file
+ Key ftCrypto.TransferKey // 256-bit encryption key
+ Mac []byte // 256-bit MAC of the entire file
+ NumParts uint16 // Number of file parts
+ Size uint32 // The size of the file, in bytes
+ Retry float32 // Determines how many times to retry sending
+ Preview []byte // A preview of the file
+}
+
+func (ti *TransferInfo) Marshal() ([]byte, error) {
+ return json.Marshal(ti)
+}
+
+func UnmarshalTransferInfo(data []byte) (*TransferInfo, error) {
+ var ti TransferInfo
+ return &ti, json.Unmarshal(data, &ti)
+}
diff --git a/fileTransfer2/info_test.go b/fileTransfer2/info_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..5713698639d9e122370e8f86e9533fba6da9ea18
--- /dev/null
+++ b/fileTransfer2/info_test.go
@@ -0,0 +1,44 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileTransfer2
+
+import (
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "reflect"
+ "testing"
+)
+
+// Tests that a TransferInfo marshalled via TransferInfo.Marshal and
+// unmarshalled via UnmarshalTransferInfo matches the original.
+func TestTransferInfo_Marshal_UnmarshalTransferInfo(t *testing.T) {
+ ti := &TransferInfo{
+ FileName: "FileName",
+ FileType: "FileType",
+ Key: ftCrypto.TransferKey{1, 2, 3},
+ Mac: []byte("I am a MAC"),
+ NumParts: 6,
+ Size: 250,
+ Retry: 2.6,
+ Preview: []byte("I am a preview"),
+ }
+
+ data, err := ti.Marshal()
+ if err != nil {
+ t.Errorf("Failed to marshal TransferInfo: %+v", err)
+ }
+
+ newTi, err := UnmarshalTransferInfo(data)
+ if err != nil {
+ t.Errorf("Failed to unmarshal TransferInfo: %+v", err)
+ }
+
+ if !reflect.DeepEqual(ti, newTi) {
+ t.Errorf("Unmarshalled TransferInfo does not match original."+
+ "\nexpected: %+v\nreceived: %+v", ti, newTi)
+ }
+}
diff --git a/fileTransfer2/interface.go b/fileTransfer2/interface.go
new file mode 100644
index 0000000000000000000000000000000000000000..260e33f34037fcd06e86f9a1f23c7df0c003c9a9
--- /dev/null
+++ b/fileTransfer2/interface.go
@@ -0,0 +1,247 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileTransfer2
+
+import (
+ "gitlab.com/elixxir/client/stoppable"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/xx_network/primitives/id"
+ "strconv"
+ "time"
+)
+
+// SentProgressCallback is a callback function that tracks the progress of
+// sending a file.
+type SentProgressCallback func(completed bool, arrived, total uint16,
+ t FilePartTracker, err error)
+
+// ReceivedProgressCallback is a callback function that tracks the progress of
+// receiving a file.
+type ReceivedProgressCallback func(completed bool, received, total uint16,
+ t FilePartTracker, err error)
+
+// ReceiveCallback is a callback function that notifies the receiver of an
+// incoming file transfer.
+type ReceiveCallback func(tid *ftCrypto.TransferID, fileName, fileType string,
+ sender *id.ID, size uint32, preview []byte)
+
+// SendNew handles the sending of the initial message to the recipient informing
+// them of the incoming file transfer parts.
+type SendNew func(recipient *id.ID, info *TransferInfo)
+
+// SendEnd handles the sending of the last message to the recipient informing
+// them that the file transfer has completed.
+type SendEnd func(recipient *id.ID)
+
+// FileTransfer facilities the sending and receiving of large file transfers.
+// It allows for progress tracking of both inbound and outbound transfers.
+type FileTransfer interface {
+
+ // StartProcesses starts the listening for new file transfer messages and
+ // starts the sending threads that wait for transfers to send.
+ StartProcesses() (stoppable.Stoppable, error)
+
+ // MaxFileNameLen returns the max number of bytes allowed for a file name.
+ MaxFileNameLen() int
+
+ // MaxFileTypeLen returns the max number of bytes allowed for a file type.
+ MaxFileTypeLen() int
+
+ // MaxFileSize returns the max number of bytes allowed for a file.
+ MaxFileSize() int
+
+ // MaxPreviewSize returns the max number of bytes allowed for a file
+ // preview.
+ MaxPreviewSize() int
+
+ /* === Sending ========================================================== */
+ /* The processes of sending a file involves three main steps:
+ 1. Sending the file using Send
+ 2. Receiving transfer progress
+ 3. Closing a finished send using CloseSend
+
+ Once the file is sent, it is broken into individual, equal-length parts
+ and sent to the recipient. Every time one of these parts arrives, it is
+ reported on all registered SentProgressCallbacks for that transfer.
+
+ A SentProgressCallback is registered on the initial send. However, if the
+ client is closed and reopened, the callback must be registered again
+ using RegisterSentProgressCallback, otherwise the continued progress of
+ the transfer will not be reported.
+
+ Once the SentProgressCallback returns that the file has completed
+ sending, the file can be closed using CloseSend. If the callback reports
+ an error, then the file should also be closed using CloseSend.
+ */
+
+ // Send initiates the sending of a file to the recipient and returns a
+ // transfer ID that uniquely identifies this file transfer.
+ //
+ // In-progress transfers are restored when closing and reopening; however, a
+ // SentProgressCallback must be registered again.
+ //
+ // fileName - Human-readable file name. Max length defined by
+ // MaxFileNameLen.
+ // fileType - Shorthand that identifies the type of file. Max length
+ // defined by MaxFileTypeLen.
+ // fileData - File contents. Max size defined by MaxFileSize.
+ // recipient - ID of the receiver of the file transfer. The sender must
+ // have an E2E relationship with the recipient.
+ // retry - The number of sending retries allowed on send failure (e.g.
+ // a retry of 2.0 with 6 parts means 12 total possible sends).
+ // preview - A preview of the file data (e.g. a thumbnail). Max size
+ // defined by MaxPreviewSize.
+ // progressCB - A callback that reports the progress of the file transfer.
+ // The callback is called once on initialization, on every progress
+ // update (or less if restricted by the period), or on fatal error.
+ // period - A progress callback will be limited from triggering only once
+ // per period.
+ Send(fileName, fileType string, fileData []byte, recipient *id.ID,
+ retry float32, preview []byte, progressCB SentProgressCallback,
+ period time.Duration) (*ftCrypto.TransferID, error)
+
+ // RegisterSentProgressCallback allows for the registration of a callback to
+ // track the progress of an individual sent file transfer.
+ // SentProgressCallback is auto registered on Send; this function should be
+ // called when resuming clients or registering extra callbacks.
+ //
+ // The callback will be called immediately when added to report the current
+ // progress of the transfer. It will then call every time a file part
+ // arrives, the transfer completes, or a fatal error occurs. It is called at
+ // most once every period regardless of the number of progress updates.
+ //
+ // In the event that the client is closed and resumed, this function must be
+ // used to re-register any callbacks previously registered with this
+ // function or Send.
+ RegisterSentProgressCallback(tid *ftCrypto.TransferID,
+ progressCB SentProgressCallback, period time.Duration) error
+
+ // CloseSend deletes a file from the internal storage once a transfer has
+ // completed or reached the retry limit. Returns an error if the transfer
+ // has not run out of retries.
+ //
+ // This function should be called once a transfer completes or errors out
+ // (as reported by the progress callback).
+ CloseSend(tid *ftCrypto.TransferID) error
+
+ /* === Receiving ======================================================== */
+ /* The processes of receiving a file involves three main steps:
+ 1. Receiving a new file transfer on ReceiveCallback
+ 2. Receiving transfer progress
+ 3. Receiving the complete file using Receive
+
+ Once the file transfer manager has started, it will call the
+ ReceiveCallback for every new file transfer that is received. Once that
+ happens, a ReceivedProgressCallback must be registered using
+ RegisterReceivedProgressCallback to get progress updates on the transfer.
+
+ When the progress callback reports that the transfer is complete, the
+ full file can be retrieved using Receive.
+ */
+
+ // AddNew starts tracking received file parts for the given file
+ // information returns a transfer ID that uniquely identifies this file
+ // transfer.
+ //
+ // This function should be called once for every new file received on the
+ // registered SendNew callback.
+ //
+ // In-progress transfers are restored when closing and reopening; however, a
+ // ReceivedProgressCallback must be registered again.
+ //
+ // progressCB - A callback that reports the progress of the file transfer.
+ // The callback is called once on initialization, on every progress
+ // update (or less if restricted by the period), or on fatal error.
+ // period - A progress callback will be limited from triggering only once
+ // per period.
+ AddNew(fileName string, key *ftCrypto.TransferKey, transferMAC []byte,
+ numParts uint16, size uint32, retry float32,
+ progressCB ReceivedProgressCallback, period time.Duration) (
+ *ftCrypto.TransferID, error)
+
+ // RegisterReceivedProgressCallback allows for the registration of a
+ // callback to track the progress of an individual received file transfer.
+ // This should be done when a new transfer is received on the
+ // ReceiveCallback.
+ //
+ // The callback will be called immediately when added to report the current
+ // progress of the transfer. It will then call every time a file part is
+ // received, the transfer completes, or a fatal error occurs. It is called
+ // at most once every period regardless of the number of progress updates.
+ //
+ // In the event that the client is closed and resumed, this function must be
+ // used to re-register any callbacks previously registered.
+ //
+ // Once the callback reports that the transfer has completed, the recipient
+ // can get the full file by calling Receive.
+ RegisterReceivedProgressCallback(tid *ftCrypto.TransferID,
+ progressCB ReceivedProgressCallback, period time.Duration) error
+
+ // Receive returns the full file on the completion of the transfer.
+ // It deletes internal references to the data and unregisters any attached
+ // progress callback. Returns an error if the transfer is not complete, the
+ // full file cannot be verified, or if the transfer cannot be found.
+ //
+ // Receive can only be called once the progress callback returns that the
+ // file transfer is complete.
+ Receive(tid *ftCrypto.TransferID) ([]byte, error)
+}
+
+// FilePartTracker tracks the status of each file part in a sent or received
+// file transfer.
+type FilePartTracker interface {
+ // GetPartStatus returns the status of the file part with the given part
+ // number. The possible values for the status are:
+ // 0 < Part does not exist
+ // 0 = unsent
+ // 1 = arrived (sender has sent a part, and it has arrived)
+ // 2 = received (receiver has received a part)
+ GetPartStatus(partNum uint16) FpStatus
+
+ // GetNumParts returns the total number of file parts in the transfer.
+ GetNumParts() uint16
+}
+
+// FpStatus is the file part status and indicates the status of individual file
+// parts in a file transfer.
+type FpStatus int
+
+// Possible values for FpStatus.
+const (
+ // FpUnsent indicates that the file part has not been sent
+ FpUnsent FpStatus = iota
+
+ // FpSent indicates that the file part has been sent (sender has sent a
+ // part, but it has not arrived)
+ FpSent
+
+ // FpArrived indicates that the file part has arrived (sender has sent a
+ // part, and it has arrived)
+ FpArrived
+
+ // FpReceived indicates that the file part has been received (receiver has
+ // received a part)
+ FpReceived
+)
+
+// String returns the string representing of the FpStatus. This functions
+// satisfies the fmt.Stringer interface.
+func (fps FpStatus) String() string {
+ switch fps {
+ case FpUnsent:
+ return "unsent"
+ case FpSent:
+ return "sent"
+ case FpArrived:
+ return "arrived"
+ case FpReceived:
+ return "received"
+ default:
+ return "INVALID FpStatus: " + strconv.Itoa(int(fps))
+ }
+}
diff --git a/fileTransfer2/manager.go b/fileTransfer2/manager.go
new file mode 100644
index 0000000000000000000000000000000000000000..c44f5d8f9682e98dae10daf899e2f7d228ba4a7c
--- /dev/null
+++ b/fileTransfer2/manager.go
@@ -0,0 +1,560 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileTransfer2
+
+import (
+ "bytes"
+ "github.com/pkg/errors"
+ jww "github.com/spf13/jwalterweatherman"
+ "gitlab.com/elixxir/client/cmix"
+ "gitlab.com/elixxir/client/cmix/message"
+ "gitlab.com/elixxir/client/fileTransfer2/callbackTracker"
+ "gitlab.com/elixxir/client/fileTransfer2/store"
+ "gitlab.com/elixxir/client/fileTransfer2/store/fileMessage"
+ "gitlab.com/elixxir/client/stoppable"
+ "gitlab.com/elixxir/client/storage/versioned"
+ "gitlab.com/elixxir/crypto/fastRNG"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/elixxir/primitives/format"
+ "gitlab.com/xx_network/primitives/id"
+ "gitlab.com/xx_network/primitives/id/ephemeral"
+ "time"
+)
+
+const (
+ // FileNameMaxLen is the maximum size, in bytes, for a file name. Currently,
+ // it is set to 48 bytes.
+ FileNameMaxLen = 48
+
+ // FileTypeMaxLen is the maximum size, in bytes, for a file type. Currently,
+ // it is set to 8 bytes.
+ FileTypeMaxLen = 8
+
+ // FileMaxSize is the maximum file size that can be transferred. Currently,
+ // it is set to 250 kB.
+ FileMaxSize = 250_000
+
+ // PreviewMaxSize is the maximum size, in bytes, for a file preview.
+ // Currently, it is set to 4 kB.
+ PreviewMaxSize = 4_000
+
+ // minPartsSendPerRound is the minimum number of file parts sent each round.
+ minPartsSendPerRound = 1
+
+ // maxPartsSendPerRound is the maximum number of file parts sent each round.
+ maxPartsSendPerRound = 11
+
+ // Size of the buffered channel that queues file parts to package
+ batchQueueBuffLen = 10_000
+
+ // Size of the buffered channel that queues file packets to send
+ sendQueueBuffLen = 10_000
+)
+
+// Stoppable and listener values.
+const (
+ fileTransferStoppable = "FileTransfer"
+ workerPoolStoppable = "FilePartSendingWorkerPool"
+ batchBuilderThreadStoppable = "BatchBuilderThread"
+)
+
+// Error messages.
+const (
+ errNoSentTransfer = "could not find sent transfer with ID %s"
+ errNoReceivedTransfer = "could not find received transfer with ID %s"
+
+ // NewManager
+ errNewOrLoadSent = "failed to load or create new list of sent file transfers: %+v"
+ errNewOrLoadReceived = "failed to load or create new list of received file transfers: %+v"
+
+ // manager.Send
+ errFileNameSize = "length of filename (%d) greater than max allowed length (%d)"
+ errFileTypeSize = "length of file type (%d) greater than max allowed length (%d)"
+ errFileSize = "size of file (%d bytes) greater than max allowed size (%d bytes)"
+ errPreviewSize = "size of preview (%d bytes) greater than max allowed size (%d bytes)"
+ errSendNetworkHealth = "cannot initiate file transfer of %q when network is not healthy."
+ errNewKey = "could not generate new transfer key: %+v"
+ errNewID = "could not generate new transfer ID: %+v"
+ errAddSentTransfer = "failed to add transfer: %+v"
+
+ // manager.CloseSend
+ errDeleteIncompleteTransfer = "cannot delete transfer %s that has not completed or failed"
+ errDeleteSentTransfer = "could not delete sent transfer %s: %+v"
+ errRemoveSentTransfer = "could not remove transfer %s from list: %+v"
+
+ // manager.AddNew
+ errNewRtTransferID = "failed to generate transfer ID for new received file transfer %q: %+v"
+ errAddNewRt = "failed to add new file transfer %s (%q): %+v"
+
+ // manager.Receive
+ errIncompleteFile = "cannot get incomplete file: missing %d of %d parts"
+ errDeleteReceivedTransfer = "could not delete received transfer %s: %+v"
+ errRemoveReceivedTransfer = "could not remove transfer %s from list: %+v"
+)
+
+// manager handles the sending and receiving of file, their storage, and their
+// callbacks.
+type manager struct {
+ // Storage-backed structure for tracking sent file transfers
+ sent *store.Sent
+
+ // Storage-backed structure for tracking received file transfers
+ received *store.Received
+
+ // Progress callback tracker
+ callbacks *callbackTracker.Manager
+
+ // Queue of parts to batch and send
+ batchQueue chan store.Part
+
+ // Queue of batches of parts to send
+ sendQueue chan []store.Part
+
+ // Function to call to send new file transfer information to recipient
+ sendNewCb SendNew
+
+ // Function to call to send notification that file transfer has completed
+ sendEndCb SendEnd
+
+ // File transfer parameters
+ params Params
+
+ myID *id.ID
+ cmix Cmix
+ kv *versioned.KV
+ rng *fastRNG.StreamGenerator
+}
+
+type Cmix interface {
+ GetMaxMessageLength() int
+ SendMany(messages []cmix.TargetedCmixMessage, p cmix.CMIXParams) (id.Round,
+ []ephemeral.Id, error)
+ AddFingerprint(identity *id.ID, fingerprint format.Fingerprint,
+ mp message.Processor) error
+ DeleteFingerprint(identity *id.ID, fingerprint format.Fingerprint)
+ IsHealthy() bool
+ AddHealthCallback(f func(bool)) uint64
+ RemoveHealthCallback(uint64)
+ GetRoundResults(timeout time.Duration, roundCallback cmix.RoundEventCallback,
+ roundList ...id.Round) error
+}
+
+// NewManager creates a new file transfer manager object. If sent or received
+// transfers already existed, they are loaded from storage and queued to resume
+// once manager.startProcesses is called.
+func NewManager(sendNewCb SendNew, sendEndCb SendEnd, params Params,
+ myID *id.ID, cmix Cmix, kv *versioned.KV,
+ rng *fastRNG.StreamGenerator) (FileTransfer, error) {
+
+ // Create a new list of sent file transfers or load one if it exists
+ sent, unsentParts, err := store.NewOrLoadSent(kv)
+ if err != nil {
+ return nil, errors.Errorf(errNewOrLoadSent, err)
+ }
+
+ // Create a new list of received file transfers or load one if it exists
+ received, incompleteTransfers, err := store.NewOrLoadReceived(kv)
+ if err != nil {
+ return nil, errors.Errorf(errNewOrLoadReceived, err)
+ }
+
+ // Construct manager
+ m := &manager{
+ sent: sent,
+ received: received,
+ callbacks: callbackTracker.NewManager(),
+ batchQueue: make(chan store.Part, batchQueueBuffLen),
+ sendQueue: make(chan []store.Part, sendQueueBuffLen),
+ sendNewCb: sendNewCb,
+ sendEndCb: sendEndCb,
+ params: params,
+ myID: myID,
+ cmix: cmix,
+ kv: kv,
+ rng: rng,
+ }
+
+ // Add all unsent file parts to queue
+ for _, p := range unsentParts {
+ m.batchQueue <- p
+ }
+
+ // Add all fingerprints for unreceived parts
+ for _, rt := range incompleteTransfers {
+ m.addFingerprints(rt)
+ }
+
+ return m, nil
+}
+
+// StartProcesses starts the sending threads. Adheres to the api.Service type.
+func (m *manager) StartProcesses() (stoppable.Stoppable, error) {
+
+ // Construct stoppables
+ multiStop := stoppable.NewMulti(workerPoolStoppable)
+ batchBuilderStop := stoppable.NewSingle(batchBuilderThreadStoppable)
+
+ // Start sending threads
+ go m.startSendingWorkerPool(multiStop)
+ go m.batchBuilderThread(batchBuilderStop)
+
+ // Create a multi stoppable
+ multiStoppable := stoppable.NewMulti(fileTransferStoppable)
+ multiStoppable.Add(multiStop)
+ multiStoppable.Add(batchBuilderStop)
+
+ return multiStoppable, nil
+}
+
+// MaxFileNameLen returns the max number of bytes allowed for a file name.
+func (m *manager) MaxFileNameLen() int {
+ return FileNameMaxLen
+}
+
+// MaxFileTypeLen returns the max number of bytes allowed for a file type.
+func (m *manager) MaxFileTypeLen() int {
+ return FileTypeMaxLen
+}
+
+// MaxFileSize returns the max number of bytes allowed for a file.
+func (m *manager) MaxFileSize() int {
+ return FileMaxSize
+}
+
+// MaxPreviewSize returns the max number of bytes allowed for a file preview.
+func (m *manager) MaxPreviewSize() int {
+ return PreviewMaxSize
+}
+
+/* === Sending ============================================================== */
+
+// Send partitions the given file into cMix message sized chunks and sends them
+// via cmix.SendMany.
+func (m *manager) Send(fileName, fileType string, fileData []byte,
+ recipient *id.ID, retry float32, preview []byte,
+ progressCB SentProgressCallback, period time.Duration) (
+ *ftCrypto.TransferID, error) {
+
+ // Return an error if the file name is too long
+ if len(fileName) > FileNameMaxLen {
+ return nil, errors.Errorf(errFileNameSize, len(fileName), FileNameMaxLen)
+ }
+
+ // Return an error if the file type is too long
+ if len(fileType) > FileTypeMaxLen {
+ return nil, errors.Errorf(errFileTypeSize, len(fileType), FileTypeMaxLen)
+ }
+
+ // Return an error if the file is too large
+ if len(fileData) > FileMaxSize {
+ return nil, errors.Errorf(errFileSize, len(fileData), FileMaxSize)
+ }
+
+ // Return an error if the preview is too large
+ if len(preview) > PreviewMaxSize {
+ return nil, errors.Errorf(errPreviewSize, len(preview), PreviewMaxSize)
+ }
+
+ // Return an error if the network is not healthy
+ if !m.cmix.IsHealthy() {
+ return nil, errors.Errorf(errSendNetworkHealth, fileName)
+ }
+
+ // Generate new transfer key and transfer ID
+ rng := m.rng.GetStream()
+ key, err := ftCrypto.NewTransferKey(rng)
+ if err != nil {
+ rng.Close()
+ return nil, errors.Errorf(errNewKey, err)
+ }
+ tid, err := ftCrypto.NewTransferID(rng)
+ if err != nil {
+ rng.Close()
+ return nil, errors.Errorf(errNewID, err)
+ }
+ rng.Close()
+
+ // Generate transfer MAC
+ mac := ftCrypto.CreateTransferMAC(fileData, key)
+
+ // Get size of each part and partition file into equal length parts
+ partMessage := fileMessage.NewPartMessage(m.cmix.GetMaxMessageLength())
+ parts := partitionFile(fileData, partMessage.GetPartSize())
+ numParts := uint16(len(parts))
+ fileSize := uint32(len(fileData))
+
+ // Send the initial file transfer message over E2E
+ info := &TransferInfo{
+ FileName: fileName,
+ FileType: fileType,
+ Key: key,
+ Mac: mac,
+ NumParts: numParts,
+ Size: fileSize,
+ Retry: retry,
+ Preview: preview,
+ }
+ go m.sendNewCb(recipient, info)
+
+ // Calculate the number of fingerprints to generate
+ numFps := calcNumberOfFingerprints(len(parts), retry)
+
+ // Create new sent transfer
+ st, err := m.sent.AddTransfer(recipient, &key, &tid, fileName, parts, numFps)
+ if err != nil {
+ return nil, errors.Errorf(errAddSentTransfer, err)
+ }
+
+ // Add all parts to the send queue
+ for _, p := range st.GetUnsentParts() {
+ m.batchQueue <- p
+ }
+
+ // Register the progress callback
+ m.registerSentProgressCallback(st, progressCB, period)
+
+ return &tid, nil
+}
+
+// RegisterSentProgressCallback adds the given callback to the callback manager
+// for the given transfer ID. Returns an error if the transfer cannot be found.
+func (m *manager) RegisterSentProgressCallback(tid *ftCrypto.TransferID,
+ progressCB SentProgressCallback, period time.Duration) error {
+ st, exists := m.sent.GetTransfer(tid)
+ if !exists {
+ return errors.Errorf(errNoSentTransfer, tid)
+ }
+
+ m.registerSentProgressCallback(st, progressCB, period)
+
+ return nil
+}
+
+// registerSentProgressCallback creates a callback for the sent transfer that
+// will get the most recent progress and send it on the progress callback.
+func (m *manager) registerSentProgressCallback(st *store.SentTransfer,
+ progressCB SentProgressCallback, period time.Duration) {
+ if progressCB == nil {
+ return
+ }
+
+ // Build callback
+ cb := func(err error) {
+ // Get transfer progress
+ arrived, total := st.NumArrived(), st.NumParts()
+ completed := arrived == total
+
+ // If the transfer is completed, send last message informing recipient
+ if completed && m.params.NotifyUponCompletion {
+ go m.sendEndCb(st.Recipient())
+ }
+
+ // Build part tracker from copy of part statuses vector
+ tracker := &sentFilePartTracker{st.CopyPartStatusVector()}
+
+ // Call the progress callback
+ progressCB(completed, arrived, total, tracker, err)
+ }
+
+ // Add the callback to the callback tracker
+ m.callbacks.AddCallback(st.TransferID(), cb, period)
+}
+
+// CloseSend deletes the sent transfer from storage and the sent transfer list.
+// Also stops any scheduled progress callbacks and deletes them from the manager
+// to prevent any further calls. Deletion only occurs if the transfer has either
+// completed or failed.
+func (m *manager) CloseSend(tid *ftCrypto.TransferID) error {
+ st, exists := m.sent.GetTransfer(tid)
+ if !exists {
+ return errors.Errorf(errNoSentTransfer, tid)
+ }
+
+ // Check that the transfer is either completed or failed
+ if st.Status() != store.Completed && st.Status() != store.Failed {
+ return errors.Errorf(errDeleteIncompleteTransfer, tid)
+ }
+
+ // Delete from storage
+ err := st.Delete()
+ if err != nil {
+ return errors.Errorf(errDeleteSentTransfer, tid, err)
+ }
+
+ // Delete from transfers list
+ err = m.sent.RemoveTransfer(tid)
+ if err != nil {
+ return errors.Errorf(errRemoveSentTransfer, tid, err)
+ }
+
+ // Stop and delete all progress callbacks
+ m.callbacks.Delete(tid)
+
+ return nil
+}
+
+/* === Receiving ============================================================ */
+
+func (m *manager) AddNew(fileName string, key *ftCrypto.TransferKey,
+ transferMAC []byte, numParts uint16, size uint32, retry float32,
+ progressCB ReceivedProgressCallback, period time.Duration) (
+ *ftCrypto.TransferID, error) {
+
+ // Generate new transfer ID
+ rng := m.rng.GetStream()
+ tid, err := ftCrypto.NewTransferID(rng)
+ if err != nil {
+ rng.Close()
+ return nil, errors.Errorf(errNewRtTransferID, fileName, err)
+ }
+ rng.Close()
+
+ // Calculate the number of fingerprints based on the retry rate
+ numFps := calcNumberOfFingerprints(int(numParts), retry)
+
+ // Store the transfer
+ rt, err := m.received.AddTransfer(
+ key, &tid, fileName, transferMAC, numParts, numFps, size)
+ if err != nil {
+ return nil, errors.Errorf(errAddNewRt, tid, fileName, err)
+ }
+
+ // Start tracking fingerprints for each file part
+ m.addFingerprints(rt)
+
+ // Register the progress callback
+ m.registerReceivedProgressCallback(rt, progressCB, period)
+
+ return &tid, nil
+}
+
+// Receive concatenates the received file and returns it. Only returns the file
+// if all file parts have been received and returns an error otherwise. Also
+// deletes the transfer from storage. Once Receive has been called on a file, it
+// cannot be received again.
+func (m *manager) Receive(tid *ftCrypto.TransferID) ([]byte, error) {
+ rt, exists := m.received.GetTransfer(tid)
+ if !exists {
+ return nil, errors.Errorf(errNoReceivedTransfer, tid)
+ }
+
+ // Return an error if the transfer is not complete
+ if rt.NumReceived() != rt.NumParts() {
+ return nil, errors.Errorf(
+ errIncompleteFile, rt.NumParts()-rt.NumReceived(), rt.NumParts())
+ }
+
+ // Get the file
+ file := rt.GetFile()
+
+ // Delete all unused fingerprints
+ for _, c := range rt.GetUnusedCyphers() {
+ m.cmix.DeleteFingerprint(m.myID, c.GetFingerprint())
+ }
+
+ // Delete from storage
+ err := rt.Delete()
+ if err != nil {
+ return nil, errors.Errorf(errDeleteReceivedTransfer, tid, err)
+ }
+
+ // Delete from transfers list
+ err = m.received.RemoveTransfer(tid)
+ if err != nil {
+ return nil, errors.Errorf(errRemoveReceivedTransfer, tid, err)
+ }
+
+ // Stop and delete all progress callbacks
+ m.callbacks.Delete(tid)
+
+ return file, nil
+}
+
+// RegisterReceivedProgressCallback adds the given callback to the callback
+// manager for the given transfer ID. Returns an error if the transfer cannot be
+// found.
+func (m *manager) RegisterReceivedProgressCallback(tid *ftCrypto.TransferID,
+ progressCB ReceivedProgressCallback, period time.Duration) error {
+ rt, exists := m.received.GetTransfer(tid)
+ if !exists {
+ return errors.Errorf(errNoReceivedTransfer, tid)
+ }
+
+ m.registerReceivedProgressCallback(rt, progressCB, period)
+
+ return nil
+}
+
+// registerReceivedProgressCallback creates a callback for the received transfer
+// that will get the most recent progress and send it on the progress callback.
+func (m *manager) registerReceivedProgressCallback(rt *store.ReceivedTransfer,
+ progressCB ReceivedProgressCallback, period time.Duration) {
+ if progressCB == nil {
+ return
+ }
+
+ // Build callback
+ cb := func(err error) {
+ // Get transfer progress
+ received, total := rt.NumReceived(), rt.NumParts()
+ completed := received == total
+
+ // Build part tracker from copy of part statuses vector
+ tracker := &receivedFilePartTracker{rt.CopyPartStatusVector()}
+
+ // Call the progress callback
+ progressCB(completed, received, total, tracker, err)
+ }
+
+ // Add the callback to the callback tracker
+ m.callbacks.AddCallback(rt.TransferID(), cb, period)
+}
+
+/* === Utility ============================================================== */
+
+// partitionFile splits the file into parts of the specified part size.
+func partitionFile(file []byte, partSize int) [][]byte {
+ // Initialize part list to the correct size
+ numParts := (len(file) + partSize - 1) / partSize
+ parts := make([][]byte, 0, numParts)
+ buff := bytes.NewBuffer(file)
+
+ for n := buff.Next(partSize); len(n) > 0; n = buff.Next(partSize) {
+ newPart := make([]byte, partSize)
+ copy(newPart, n)
+ parts = append(parts, newPart)
+ }
+
+ return parts
+}
+
+// calcNumberOfFingerprints is the formula used to calculate the number of
+// fingerprints to generate, which is based off the number of file parts and the
+// retry float.
+func calcNumberOfFingerprints(numParts int, retry float32) uint16 {
+ return uint16(float32(numParts) * (1 + retry))
+}
+
+// addFingerprints adds all fingerprints for unreceived parts in the received
+// transfer.
+func (m *manager) addFingerprints(rt *store.ReceivedTransfer) {
+ // Build processor for each file part and add its fingerprint to receive on
+ for _, c := range rt.GetUnusedCyphers() {
+ p := &processor{
+ Cypher: c,
+ ReceivedTransfer: rt,
+ manager: m,
+ }
+
+ err := m.cmix.AddFingerprint(m.myID, c.GetFingerprint(), p)
+ if err != nil {
+ jww.ERROR.Printf("[FT] Failed to add fingerprint for transfer "+
+ "%s: %+v", rt.TransferID(), err)
+ }
+ }
+}
diff --git a/fileTransfer2/manager_test.go b/fileTransfer2/manager_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..04fddb3e274734d932fd0b34d166595c023d734a
--- /dev/null
+++ b/fileTransfer2/manager_test.go
@@ -0,0 +1,236 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileTransfer2
+
+import (
+ "bytes"
+ "gitlab.com/elixxir/client/cmix"
+ "gitlab.com/elixxir/client/storage/versioned"
+ "gitlab.com/elixxir/crypto/fastRNG"
+ "gitlab.com/elixxir/ekv"
+ "gitlab.com/xx_network/crypto/csprng"
+ "gitlab.com/xx_network/primitives/id"
+ "gitlab.com/xx_network/primitives/netTime"
+ "math"
+ "math/rand"
+ "reflect"
+ "sync"
+ "testing"
+ "time"
+)
+
+// Tests that manager adheres to the FileTransfer interface.
+var _ FileTransfer = (*manager)(nil)
+
+// Tests that Cmix adheres to the cmix.Client interface.
+var _ Cmix = (cmix.Client)(nil)
+
+// Tests that partitionFile partitions the given file into the expected parts.
+func Test_partitionFile(t *testing.T) {
+ prng := rand.New(rand.NewSource(42))
+ partSize := 96
+ fileData, expectedParts := newFile(24, partSize, prng, t)
+
+ receivedParts := partitionFile(fileData, partSize)
+
+ if !reflect.DeepEqual(expectedParts, receivedParts) {
+ t.Errorf("File parts do not match expected."+
+ "\nexpected: %q\nreceived: %q", expectedParts, receivedParts)
+ }
+
+ fullFile := bytes.Join(receivedParts, nil)
+ if !bytes.Equal(fileData, fullFile) {
+ t.Errorf("Full file does not match expected."+
+ "\nexpected: %q\nreceived: %q", fileData, fullFile)
+ }
+}
+
+// Tests that calcNumberOfFingerprints matches some manually calculated results.
+func Test_calcNumberOfFingerprints(t *testing.T) {
+ testValues := []struct {
+ numParts int
+ retry float32
+ result uint16
+ }{
+ {12, 0.5, 18},
+ {13, 0.6667, 21},
+ {1, 0.89, 1},
+ {2, 0.75, 3},
+ {119, 0.45, 172},
+ }
+
+ for i, val := range testValues {
+ result := calcNumberOfFingerprints(val.numParts, val.retry)
+
+ if val.result != result {
+ t.Errorf("calcNumberOfFingerprints(%3d, %3.2f) result is "+
+ "incorrect (%d).\nexpected: %d\nreceived: %d",
+ val.numParts, val.retry, i, val.result, result)
+ }
+ }
+}
+
+// Smoke test of the entire file transfer system.
+func Test_FileTransfer_Smoke(t *testing.T) {
+ // jww.SetStdoutThreshold(jww.LevelDebug)
+ // Set up cMix and E2E message handlers
+ cMixHandler := newMockCmixHandler()
+ rngGen := fastRNG.NewStreamGenerator(1000, 10, csprng.NewSystemRNG)
+ params := DefaultParams()
+ params.MaxThroughput = math.MaxInt
+
+ // Set up the first client
+ myID1 := id.NewIdFromString("myID1", id.User, t)
+ kv1 := versioned.NewKV(ekv.MakeMemstore())
+ sendNewCbChan1 := make(chan *TransferInfo)
+ sendNewCb1 := func(recipient *id.ID, info *TransferInfo) { sendNewCbChan1 <- info }
+ sendEndCbChan1 := make(chan *id.ID)
+ sendEndCb1 := func(recipient *id.ID) { sendEndCbChan1 <- recipient }
+ ftm1, err := NewManager(sendNewCb1, sendEndCb1, params, myID1,
+ newMockCmix(myID1, cMixHandler), kv1, rngGen)
+ if err != nil {
+ t.Errorf("Failed to create new file transfer manager 1: %+v", err)
+ }
+ m1 := ftm1.(*manager)
+
+ stop1, err := m1.StartProcesses()
+ if err != nil {
+ t.Errorf("Failed to start processes for manager 1: %+v", err)
+ }
+
+ // Set up the second client
+ myID2 := id.NewIdFromString("myID2", id.User, t)
+ kv2 := versioned.NewKV(ekv.MakeMemstore())
+ ftm2, err := NewManager(nil, nil, params, myID2,
+ newMockCmix(myID2, cMixHandler), kv2, rngGen)
+ if err != nil {
+ t.Errorf("Failed to create new file transfer manager 2: %+v", err)
+ }
+ m2 := ftm2.(*manager)
+
+ stop2, err := m2.StartProcesses()
+ if err != nil {
+ t.Errorf("Failed to start processes for manager 2: %+v", err)
+ }
+
+ // Wait group prevents the test from quiting before the file has completed
+ // sending and receiving
+ var wg sync.WaitGroup
+
+ // Define details of file to send
+ fileName, fileType := "myFile", "txt"
+ fileData := []byte(loremIpsum)
+ preview := []byte("Lorem ipsum dolor sit amet")
+ retry := float32(2.0)
+
+ // Create go func that waits for file transfer to be received to register
+ // a progress callback that then checks that the file received is correct
+ // when done
+ wg.Add(1)
+ var called bool
+ timeReceived := make(chan time.Time)
+ go func() {
+ select {
+ case r := <-sendNewCbChan1:
+ tid, err := m2.AddNew(
+ r.FileName, &r.Key, r.Mac, r.NumParts, r.Size, r.Retry, nil, 0)
+ if err != nil {
+ t.Errorf("Failed to add transfer: %+v", err)
+ }
+ receiveProgressCB := func(completed bool, received, total uint16,
+ fpt FilePartTracker, err error) {
+ if completed && !called {
+ timeReceived <- netTime.Now()
+ receivedFile, err2 := m2.Receive(tid)
+ if err2 != nil {
+ t.Errorf("Failed to receive file: %+v", err2)
+ }
+
+ if !bytes.Equal(fileData, receivedFile) {
+ t.Errorf("Received file does not match sent."+
+ "\nsent: %q\nreceived: %q",
+ fileData, receivedFile)
+ }
+ wg.Done()
+ }
+ }
+ err3 := m2.RegisterReceivedProgressCallback(
+ tid, receiveProgressCB, 0)
+ if err3 != nil {
+ t.Errorf(
+ "Failed to Rregister received progress callback: %+v", err3)
+ }
+ case <-time.After(2100 * time.Millisecond):
+ t.Errorf("Timed out waiting to receive new file transfer.")
+ wg.Done()
+ }
+ }()
+
+ // Define sent progress callback
+ wg.Add(1)
+ sentProgressCb1 := func(completed bool, arrived, total uint16,
+ fpt FilePartTracker, err error) {
+ if completed {
+ wg.Done()
+ }
+ }
+
+ // Send file.
+ sendStart := netTime.Now()
+ tid1, err := m1.Send(
+ fileName, fileType, fileData, myID2, retry, preview, sentProgressCb1, 0)
+ if err != nil {
+ t.Errorf("Failed to send file: %+v", err)
+ }
+
+ go func() {
+ select {
+ case tr := <-timeReceived:
+ fileSize := len(fileData)
+ sendTime := tr.Sub(sendStart)
+ fileSizeKb := float32(fileSize) * .001
+ speed := fileSizeKb * float32(time.Second) / (float32(sendTime))
+ t.Logf("Completed receiving file %q in %s (%.2f kb @ %.2f kb/s).",
+ fileName, sendTime, fileSizeKb, speed)
+ }
+ }()
+
+ // Wait for file to be sent and received
+ wg.Wait()
+
+ select {
+ case <-sendEndCbChan1:
+ case <-time.After(15 * time.Millisecond):
+ t.Error("Timed out waiting for end callback to be called.")
+ }
+
+ err = m1.CloseSend(tid1)
+ if err != nil {
+ t.Errorf("Failed to close transfer: %+v", err)
+ }
+
+ err = stop1.Close()
+ if err != nil {
+ t.Errorf("Failed to close processes for manager 1: %+v", err)
+ }
+
+ err = stop2.Close()
+ if err != nil {
+ t.Errorf("Failed to close processes for manager 2: %+v", err)
+ }
+}
+
+const loremIpsum = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed sit amet urna venenatis, rutrum magna maximus, tempor orci. Cras sit amet nulla id dolor blandit commodo. Suspendisse potenti. Praesent gravida porttitor metus vel aliquam. Maecenas rutrum velit at lobortis auctor. Mauris porta blandit tempor. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Morbi volutpat posuere maximus. Nunc in augue molestie ante mattis tempor.
+
+Phasellus placerat elit eu fringilla pharetra. Vestibulum consectetur pulvinar nunc, vestibulum tincidunt felis rhoncus sit amet. Duis non dolor eleifend nibh luctus eleifend. Nunc urna odio, euismod sit amet feugiat ut, dapibus vel elit. Nulla est mauris, posuere eget enim cursus, vehicula viverra est. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque mattis, nisi quis consectetur semper, neque enim rhoncus dolor, ut aliquam leo orci sed dolor. Integer ullamcorper pulvinar turpis, a sollicitudin nunc posuere et. Nullam orci nibh, facilisis ac massa eu, bibendum bibendum sapien. Sed tincidunt nunc mauris, nec ullamcorper enim lacinia nec. Nulla dapibus sapien ut odio bibendum, tempus ornare sapien lacinia.
+
+Duis ac hendrerit augue. Nullam porttitor feugiat finibus. Nam enim urna, maximus et ligula eu, aliquet convallis turpis. Vestibulum luctus quam in dictum efficitur. Vestibulum ac pulvinar ipsum. Vivamus consectetur augue nec tellus mollis, at iaculis magna efficitur. Nunc dictum convallis sem, at vehicula nulla accumsan non. Nullam blandit orci vel turpis convallis, mollis porttitor felis accumsan. Sed non posuere leo. Proin ultricies varius nulla at ultricies. Phasellus et pharetra justo. Quisque eu orci odio. Pellentesque pharetra tempor tempor. Aliquam ac nulla lorem. Sed dignissim ligula sit amet nibh fermentum facilisis.
+
+Donec facilisis rhoncus ante. Duis nec nisi et dolor congue semper vel id ligula. Mauris non eleifend libero, et sodales urna. Nullam pharetra gravida velit non mollis. Integer vel ultrices libero, at ultrices magna. Duis semper risus a leo vulputate consectetur. Cras sit amet convallis sapien. Sed blandit, felis et porttitor fringilla, urna tellus commodo metus, at pharetra nibh urna sed sem. Nam ex dui, posuere id mi et, egestas tincidunt est. Nullam elementum pulvinar diam in maximus. Maecenas vel augue vitae nunc consectetur vestibulum in aliquet lacus. Nullam nec lectus dapibus, dictum nisi nec, congue quam. Suspendisse mollis vel diam nec dapibus. Mauris neque justo, scelerisque et suscipit non, imperdiet eget leo. Vestibulum leo turpis, dapibus ac lorem a, mollis pulvinar quam.
+
+Sed sed mauris a neque dignissim aliquet. Aliquam congue gravida velit in efficitur. Integer elementum feugiat est, ac lacinia libero bibendum sed. Sed vestibulum suscipit dignissim. Nunc scelerisque, turpis quis varius tristique, enim lacus vehicula lacus, id vestibulum velit erat eu odio. Donec tincidunt nunc sit amet sapien varius ornare. Phasellus semper venenatis ligula eget euismod. Mauris sodales massa tempor, cursus velit a, feugiat neque. Sed odio justo, rhoncus eu fermentum non, tristique a quam. In vehicula in tortor nec iaculis. Cras ligula sem, sollicitudin at nulla eget, placerat lacinia massa. Mauris tempus quam sit amet leo efficitur egestas. Proin iaculis, velit in blandit egestas, felis odio sollicitudin ipsum, eget interdum leo odio tempor nisi. Curabitur sed mauris id turpis tempor finibus ut mollis lectus. Curabitur neque libero, aliquam facilisis lobortis eget, posuere in augue. In sodales urna sit amet elit euismod rhoncus.`
diff --git a/fileTransfer2/params.go b/fileTransfer2/params.go
new file mode 100644
index 0000000000000000000000000000000000000000..5b7292d0ce21fde9b621cd140b85062fe24c7608
--- /dev/null
+++ b/fileTransfer2/params.go
@@ -0,0 +1,41 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileTransfer2
+
+import "time"
+
+const (
+ defaultMaxThroughput = 150_000 // 150 kB per second
+ defaultSendTimeout = 500 * time.Millisecond
+ defaultNotifyUponCompletion = true
+)
+
+// Params contains parameters used for file transfer.
+type Params struct {
+ // MaxThroughput is the maximum data transfer speed to send file parts (in
+ // bytes per second)
+ MaxThroughput int
+
+ // SendTimeout is the duration, in nanoseconds, before sending on a round
+ // times out. It is recommended that SendTimeout is not changed from its
+ // default.
+ SendTimeout time.Duration
+
+ // NotifyUponCompletion indicates if a final notification message is sent
+ // to the recipient on completion of file transfer. If true, the ping is
+ NotifyUponCompletion bool
+}
+
+// DefaultParams returns a Params object filled with the default values.
+func DefaultParams() Params {
+ return Params{
+ MaxThroughput: defaultMaxThroughput,
+ SendTimeout: defaultSendTimeout,
+ NotifyUponCompletion: defaultNotifyUponCompletion,
+ }
+}
diff --git a/fileTransfer2/params_test.go b/fileTransfer2/params_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..48d91b47c2527e9cc05171f08f56565c00901424
--- /dev/null
+++ b/fileTransfer2/params_test.go
@@ -0,0 +1,28 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileTransfer2
+
+import (
+ "reflect"
+ "testing"
+)
+
+// Tests that DefaultParams returns a Params object with the expected defaults.
+func TestDefaultParams(t *testing.T) {
+ expected := Params{
+ MaxThroughput: defaultMaxThroughput,
+ SendTimeout: defaultSendTimeout,
+ NotifyUponCompletion: defaultNotifyUponCompletion,
+ }
+ received := DefaultParams()
+
+ if !reflect.DeepEqual(expected, received) {
+ t.Errorf("Received Params does not match expected."+
+ "\nexpected: %+v\nreceived: %+v", expected, received)
+ }
+}
diff --git a/fileTransfer2/partTracker.go b/fileTransfer2/partTracker.go
new file mode 100644
index 0000000000000000000000000000000000000000..56fc926428040e12b0bedcd8589b0dfb5c7cbb25
--- /dev/null
+++ b/fileTransfer2/partTracker.go
@@ -0,0 +1,68 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileTransfer2
+
+import (
+ "gitlab.com/elixxir/client/storage/utility"
+)
+
+// sentFilePartTracker contains utility.StateVector that tracks which parts have
+// arrived. It adheres to the FilePartTracker interface.
+type sentFilePartTracker struct {
+ *utility.StateVector
+}
+
+// GetPartStatus returns the status of the sent file part with the given part
+// number.
+func (s *sentFilePartTracker) GetPartStatus(partNum uint16) FpStatus {
+ if uint32(partNum) >= s.GetNumKeys() {
+ return -1
+ }
+
+ switch s.Used(uint32(partNum)) {
+ case true:
+ return FpArrived
+ case false:
+ return FpUnsent
+ default:
+ return -1
+ }
+}
+
+// GetNumParts returns the total number of file parts in the transfer.
+func (s *sentFilePartTracker) GetNumParts() uint16 {
+ return uint16(s.GetNumKeys())
+}
+
+// receivedFilePartTracker contains utility.StateVector that tracks which parts
+// have been received. It adheres to the FilePartTracker interface.
+type receivedFilePartTracker struct {
+ *utility.StateVector
+}
+
+// GetPartStatus returns the status of the received file part with the given
+// part number.
+func (r *receivedFilePartTracker) GetPartStatus(partNum uint16) FpStatus {
+ if uint32(partNum) >= r.GetNumKeys() {
+ return -1
+ }
+
+ switch r.Used(uint32(partNum)) {
+ case true:
+ return FpReceived
+ case false:
+ return FpUnsent
+ default:
+ return -1
+ }
+}
+
+// GetNumParts returns the total number of file parts in the transfer.
+func (r *receivedFilePartTracker) GetNumParts() uint16 {
+ return uint16(r.GetNumKeys())
+}
diff --git a/fileTransfer2/processor.go b/fileTransfer2/processor.go
new file mode 100644
index 0000000000000000000000000000000000000000..55f2021115ac7a45f4b5b08286dffc7a1ebaeabe
--- /dev/null
+++ b/fileTransfer2/processor.go
@@ -0,0 +1,70 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileTransfer2
+
+import (
+ "fmt"
+
+ jww "github.com/spf13/jwalterweatherman"
+ "gitlab.com/elixxir/client/cmix/identity/receptionID"
+ "gitlab.com/elixxir/client/cmix/rounds"
+ "gitlab.com/elixxir/client/fileTransfer2/store"
+ "gitlab.com/elixxir/client/fileTransfer2/store/cypher"
+ "gitlab.com/elixxir/client/fileTransfer2/store/fileMessage"
+ "gitlab.com/elixxir/primitives/format"
+)
+
+// Error messages.
+const (
+ // processor.Process
+ errDecryptPart = "[FT] Failed to decrypt file part for transfer %s (%q) on round %d: %+v"
+ errUnmarshalPart = "[FT] Failed to unmarshal decrypted file part for transfer %s (%q) on round %d: %+v"
+ errAddPart = "[FT] Failed to add part #%d to transfer transfer %s (%q): %+v"
+)
+
+// processor manages the reception of file transfer messages. Adheres to the
+// message.Processor interface.
+type processor struct {
+ cypher.Cypher
+ *store.ReceivedTransfer
+ *manager
+}
+
+// Process decrypts and hands off the file part message and adds it to the
+// correct file transfer.
+func (p *processor) Process(msg format.Message,
+ _ receptionID.EphemeralIdentity, round rounds.Round) {
+
+ decryptedPart, err := p.Decrypt(msg)
+ if err != nil {
+ jww.ERROR.Printf(
+ errDecryptPart, p.TransferID(), p.FileName(), round.ID, err)
+ return
+ }
+
+ partMsg, err := fileMessage.UnmarshalPartMessage(decryptedPart)
+ if err != nil {
+ jww.ERROR.Printf(
+ errUnmarshalPart, p.TransferID(), p.FileName(), round.ID, err)
+ return
+ }
+
+ err = p.AddPart(partMsg.GetPart(), int(partMsg.GetPartNum()))
+ if err != nil {
+ jww.WARN.Printf(
+ errAddPart, partMsg.GetPartNum(), p.TransferID(), p.FileName(), err)
+ return
+ }
+
+ // Call callback with updates
+ p.callbacks.Call(p.TransferID(), nil)
+}
+
+func (p *processor) String() string {
+ return fmt.Sprintf("FileTransfer(%s)", p.myID)
+}
diff --git a/fileTransfer2/send.go b/fileTransfer2/send.go
new file mode 100644
index 0000000000000000000000000000000000000000..7ff1dad9603585dc9d4dcde6d872b9325d1fe692
--- /dev/null
+++ b/fileTransfer2/send.go
@@ -0,0 +1,183 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileTransfer2
+
+import (
+ "github.com/pkg/errors"
+ jww "github.com/spf13/jwalterweatherman"
+ "gitlab.com/elixxir/client/cmix"
+ "gitlab.com/elixxir/client/cmix/message"
+ "gitlab.com/elixxir/client/fileTransfer2/sentRoundTracker"
+ "gitlab.com/elixxir/client/fileTransfer2/store"
+ "gitlab.com/elixxir/client/stoppable"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/xx_network/primitives/id"
+ "strconv"
+ "time"
+)
+
+// Error messages.
+const (
+ // generateRandomPacketSize
+ getRandomNumPartsRandPanic = "[FT] Failed to generate random number of file parts to send: %+v"
+
+ // manager.sendCmix
+ errNoMoreRetries = "file transfer failed: ran our of retries."
+)
+
+const (
+ // Duration to wait for round to finish before timing out.
+ roundResultsTimeout = 15 * time.Second
+
+ // Age when rounds that files were sent from are deleted from the tracker.
+ clearSentRoundsAge = 10 * time.Second
+
+ // Number of concurrent sending threads
+ workerPoolThreads = 4
+
+ // Tag that prints with cMix sending logs.
+ cMixDebugTag = "FT.Part"
+
+ // Prefix used for the name of a stoppable used for a sending thread
+ sendThreadStoppableName = "FilePartSendingThread#"
+)
+
+// startSendingWorkerPool initialises a worker pool of file part sending
+// threads.
+func (m *manager) startSendingWorkerPool(multiStop *stoppable.Multi) {
+ // Set up cMix sending parameters
+ params := cmix.GetDefaultCMIXParams()
+ params.SendTimeout = m.params.SendTimeout
+ params.ExcludedRounds = sentRoundTracker.NewManager(clearSentRoundsAge)
+ params.DebugTag = cMixDebugTag
+
+ for i := 0; i < workerPoolThreads; i++ {
+ stop := stoppable.NewSingle(sendThreadStoppableName + strconv.Itoa(i))
+ multiStop.Add(stop)
+ go m.sendingThread(params, stop)
+ }
+}
+
+// sendingThread sends part packets that become available oin the send queue.
+func (m *manager) sendingThread(cMixParams cmix.CMIXParams, stop *stoppable.Single) {
+ healthChan := make(chan bool, 10)
+ healthChanID := m.cmix.AddHealthCallback(func(b bool) { healthChan <- b })
+ for {
+ select {
+ case <-stop.Quit():
+ jww.DEBUG.Printf("[FT] Stopping file part sending thread: " +
+ "stoppable triggered.")
+ m.cmix.RemoveHealthCallback(healthChanID)
+ stop.ToStopped()
+ return
+ case healthy := <-healthChan:
+ for !healthy {
+ healthy = <-healthChan
+ }
+ case packet := <-m.sendQueue:
+ m.sendCmix(packet, cMixParams)
+ }
+ }
+}
+
+// sendCmix sends the parts in the packet via Client.SendMany.
+func (m *manager) sendCmix(packet []store.Part, cMixParams cmix.CMIXParams) {
+ // validParts will contain all parts in the original packet excluding those
+ // that return an error from GetEncryptedPart
+ validParts := make([]store.Part, 0, len(packet))
+
+ // Encrypt each part and to a TargetedCmixMessage
+ messages := make([]cmix.TargetedCmixMessage, 0, len(packet))
+ for _, p := range packet {
+ encryptedPart, mac, fp, err :=
+ p.GetEncryptedPart(m.cmix.GetMaxMessageLength())
+ if err != nil {
+ jww.ERROR.Printf("[FT] File transfer %s (%q) failed: %+v",
+ p.TransferID(), p.FileName(), err)
+ m.callbacks.Call(p.TransferID(), errors.New(errNoMoreRetries))
+ continue
+ }
+
+ validParts = append(validParts, p)
+
+ messages = append(messages, cmix.TargetedCmixMessage{
+ Recipient: p.Recipient(),
+ Payload: encryptedPart,
+ Fingerprint: fp,
+ Service: message.Service{},
+ Mac: mac,
+ })
+ }
+
+ // Clear all old rounds from the sent rounds list
+ cMixParams.ExcludedRounds.(*sentRoundTracker.Manager).RemoveOldRounds()
+
+ jww.DEBUG.Printf("[FT] Sending %d file parts via SendManyCMIX",
+ len(messages))
+
+ rid, _, err := m.cmix.SendMany(messages, cMixParams)
+ if err != nil {
+ jww.WARN.Printf("[FT] Failed to send %d file parts via "+
+ "SendManyCMIX: %+v", len(messages), err)
+
+ for _, p := range validParts {
+ m.batchQueue <- p
+ }
+ }
+
+ err = m.cmix.GetRoundResults(
+ roundResultsTimeout, m.roundResultsCallback(validParts), rid)
+}
+
+// roundResultsCallback generates a network.RoundEventCallback that handles
+// all parts in the packet once the round succeeds or fails.
+func (m *manager) roundResultsCallback(packet []store.Part) cmix.RoundEventCallback {
+ // Group file parts by transfer
+ grouped := map[ftCrypto.TransferID][]store.Part{}
+ for _, p := range packet {
+ if _, exists := grouped[*p.TransferID()]; exists {
+ grouped[*p.TransferID()] = append(grouped[*p.TransferID()], p)
+ } else {
+ grouped[*p.TransferID()] = []store.Part{p}
+ }
+ }
+
+ return func(
+ allRoundsSucceeded, _ bool, rounds map[id.Round]cmix.RoundResult) {
+ // Get round ID
+ var rid id.Round
+ for rid = range rounds {
+ break
+ }
+
+ if allRoundsSucceeded {
+ jww.DEBUG.Printf("[FT] %d file parts delivered on round %d (%v)",
+ len(packet), rid, grouped)
+
+ // If the round succeeded, then mark all parts as arrived and report
+ // each transfer's progress on its progress callback
+ for tid, parts := range grouped {
+ for _, p := range parts {
+ p.MarkArrived()
+ }
+
+ // Call the progress callback after all parts have been marked
+ // so that the progress reported included all parts in the batch
+ m.callbacks.Call(&tid, nil)
+ }
+ } else {
+ jww.DEBUG.Printf("[FT] %d file parts failed on round %d (%v)",
+ len(packet), rid, grouped)
+
+ // If the round failed, then add each part into the send queue
+ for _, p := range packet {
+ m.batchQueue <- p
+ }
+ }
+ }
+}
diff --git a/fileTransfer2/sentRoundTracker/sentRoundTracker.go b/fileTransfer2/sentRoundTracker/sentRoundTracker.go
new file mode 100644
index 0000000000000000000000000000000000000000..29416fa62753623c88949cc11a667d23f5f2f709
--- /dev/null
+++ b/fileTransfer2/sentRoundTracker/sentRoundTracker.go
@@ -0,0 +1,92 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package sentRoundTracker
+
+import (
+ "gitlab.com/xx_network/primitives/id"
+ "gitlab.com/xx_network/primitives/netTime"
+ "sync"
+ "time"
+)
+
+// Manager keeps track of rounds that file parts were sent on and when
+// those rounds occurred. Rounds past the given age can be deleted manually.
+// Adheres to excludedRounds.ExcludedRounds.
+type Manager struct {
+ rounds map[id.Round]time.Time
+ age time.Duration
+ mux sync.RWMutex
+}
+
+// NewManager creates a new sent round tracker Manager.
+func NewManager(interval time.Duration) *Manager {
+ return &Manager{
+ rounds: make(map[id.Round]time.Time),
+ age: interval,
+ }
+}
+
+// RemoveOldRounds removes any rounds that are older than the max round age.
+func (srt *Manager) RemoveOldRounds() {
+ srt.mux.Lock()
+ defer srt.mux.Unlock()
+ deleteBefore := netTime.Now().Add(-srt.age)
+
+ for rid, timeStamp := range srt.rounds {
+ if timeStamp.Before(deleteBefore) {
+ delete(srt.rounds, rid)
+ }
+ }
+}
+
+// Has indicates if the round ID is in the tracker.
+func (srt *Manager) Has(rid id.Round) bool {
+ srt.mux.RLock()
+ defer srt.mux.RUnlock()
+
+ _, exists := srt.rounds[rid]
+ return exists
+}
+
+// Insert adds the round to the tracker with the current time. Returns true if
+// the round was added.
+func (srt *Manager) Insert(rid id.Round) bool {
+ timeNow := netTime.Now()
+ srt.mux.Lock()
+ defer srt.mux.Unlock()
+
+ _, exists := srt.rounds[rid]
+ if exists {
+ return false
+ }
+
+ srt.rounds[rid] = timeNow
+ return true
+}
+
+// Remove deletes a round ID from the tracker.
+func (srt *Manager) Remove(rid id.Round) {
+ srt.mux.Lock()
+ defer srt.mux.Unlock()
+ delete(srt.rounds, rid)
+}
+
+// Len returns the number of round IDs in the tracker.
+func (srt *Manager) Len() int {
+ srt.mux.RLock()
+ defer srt.mux.RUnlock()
+
+ return len(srt.rounds)
+}
diff --git a/fileTransfer2/sentRoundTracker/sentRoundTracker_test.go b/fileTransfer2/sentRoundTracker/sentRoundTracker_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..f7a2fa4d74a670754e65cfbb3dca510177dc5fe6
--- /dev/null
+++ b/fileTransfer2/sentRoundTracker/sentRoundTracker_test.go
@@ -0,0 +1,192 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package sentRoundTracker
+
+import (
+ "gitlab.com/xx_network/primitives/id"
+ "reflect"
+ "testing"
+ "time"
+)
+
+// Tests that NewManager returns the expected new Manager.
+func Test_NewSentRoundTracker(t *testing.T) {
+ interval := 10 * time.Millisecond
+ expected := &Manager{
+ rounds: make(map[id.Round]time.Time),
+ age: interval,
+ }
+
+ srt := NewManager(interval)
+
+ if !reflect.DeepEqual(expected, srt) {
+ t.Errorf("New Manager does not match expected."+
+ "\nexpected: %+v\nreceived: %+v", expected, srt)
+ }
+}
+
+// Tests that Manager.RemoveOldRounds removes only old rounds and not
+// newer rounds.
+func TestManager_RemoveOldRounds(t *testing.T) {
+ srt := NewManager(50 * time.Millisecond)
+
+ // Add odd round to tracker
+ for rid := id.Round(0); rid < 100; rid++ {
+ if rid%2 != 0 {
+ srt.Insert(rid)
+ }
+ }
+
+ time.Sleep(50 * time.Millisecond)
+
+ // Add even round to tracker
+ for rid := id.Round(0); rid < 100; rid++ {
+ if rid%2 == 0 {
+ srt.Insert(rid)
+ }
+ }
+
+ // Remove all old rounds (should be all odd rounds)
+ srt.RemoveOldRounds()
+
+ // Check that only even rounds exist
+ for rid := id.Round(0); rid < 100; rid++ {
+ if srt.Has(rid) {
+ if rid%2 != 0 {
+ t.Errorf("Round %d exists.", rid)
+ }
+ } else if rid%2 == 0 {
+ t.Errorf("Round %d does not exist.", rid)
+ }
+ }
+}
+
+// Tests that Manager.Has returns true for all the even rounds and
+// false for all odd rounds.
+func TestManager_Has(t *testing.T) {
+ srt := NewManager(0)
+
+ // Insert even rounds into the tracker
+ for rid := id.Round(0); rid < 100; rid++ {
+ if rid%2 == 0 {
+ srt.Insert(rid)
+ }
+ }
+
+ // Check that only even rounds exist
+ for rid := id.Round(0); rid < 100; rid++ {
+ if srt.Has(rid) {
+ if rid%2 != 0 {
+ t.Errorf("Round %d exists.", rid)
+ }
+ } else if rid%2 == 0 {
+ t.Errorf("Round %d does not exist.", rid)
+ }
+ }
+}
+
+// Tests that Manager.Insert adds all the expected rounds to the map and that it
+// returns true when the round does not already exist and false otherwise.
+func TestManager_Insert(t *testing.T) {
+ srt := NewManager(0)
+
+ // Insert even rounds into the tracker
+ for rid := id.Round(0); rid < 100; rid++ {
+ if rid%2 == 0 {
+ if !srt.Insert(rid) {
+ t.Errorf("Did not insert round %d.", rid)
+ }
+ }
+ }
+
+ // Check that only even rounds were added
+ for rid := id.Round(0); rid < 100; rid++ {
+ _, exists := srt.rounds[rid]
+ if exists {
+ if rid%2 != 0 {
+ t.Errorf("Round %d exists.", rid)
+ }
+ } else if rid%2 == 0 {
+ t.Errorf("Round %d does not exist.", rid)
+ }
+ }
+
+ // Check that adding a round that already exists returns false
+ if srt.Insert(0) {
+ t.Errorf("Inserted round %d.", 0)
+ }
+}
+
+// Tests that Manager.Remove removes all even rounds.
+func TestManager_Remove(t *testing.T) {
+ srt := NewManager(0)
+
+ // Add all round to tracker
+ for rid := id.Round(0); rid < 100; rid++ {
+ srt.Insert(rid)
+ }
+
+ // Remove even rounds from the tracker
+ for rid := id.Round(0); rid < 100; rid++ {
+ if rid%2 == 0 {
+ srt.Remove(rid)
+ }
+ }
+
+ // Check that only even rounds were removed
+ for rid := id.Round(0); rid < 100; rid++ {
+ _, exists := srt.rounds[rid]
+ if exists {
+ if rid%2 == 0 {
+ t.Errorf("Round %d does not exist.", rid)
+ }
+ } else if rid%2 != 0 {
+ t.Errorf("Round %d exists.", rid)
+ }
+ }
+}
+
+// Tests that Manager.Len returns the expected length when the tracker
+// is empty, filled, and then modified.
+func TestManager_Len(t *testing.T) {
+ srt := NewManager(0)
+
+ if srt.Len() != 0 {
+ t.Errorf("Length of tracker incorrect.\nexpected: %d\nreceived: %d",
+ 0, srt.Len())
+ }
+
+ // Add all round to tracker
+ for rid := id.Round(0); rid < 100; rid++ {
+ srt.Insert(rid)
+ }
+
+ if srt.Len() != 100 {
+ t.Errorf("Length of tracker incorrect.\nexpected: %d\nreceived: %d",
+ 100, srt.Len())
+ }
+
+ // Remove even rounds from the tracker
+ for rid := id.Round(0); rid < 100; rid++ {
+ if rid%2 == 0 {
+ srt.Remove(rid)
+ }
+ }
+
+ if srt.Len() != 50 {
+ t.Errorf("Length of tracker incorrect.\nexpected: %d\nreceived: %d",
+ 50, srt.Len())
+ }
+}
diff --git a/fileTransfer2/store/cypher/cypher.go b/fileTransfer2/store/cypher/cypher.go
new file mode 100644
index 0000000000000000000000000000000000000000..0c05751cac11717518a4751868fba321822946b5
--- /dev/null
+++ b/fileTransfer2/store/cypher/cypher.go
@@ -0,0 +1,59 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package cypher
+
+import (
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/elixxir/primitives/format"
+)
+
+// Cypher contains the cryptographic and identifying information needed to
+// decrypt a file part and associate it with the correct file.
+type Cypher struct {
+ *Manager
+ fpNum uint16
+}
+
+// Encrypt encrypts the file part contents and returns them along with a MAC and
+// fingerprint.
+func (c Cypher) Encrypt(contents []byte) (
+ cipherText, mac []byte, fp format.Fingerprint) {
+
+ // Generate fingerprint
+ fp = ftCrypto.GenerateFingerprint(*c.key, c.fpNum)
+
+ // Encrypt part and get MAC
+ cipherText, mac = ftCrypto.EncryptPart(*c.key, contents, c.fpNum, fp)
+
+ return cipherText, mac, fp
+}
+
+// Decrypt decrypts the content of the message.
+func (c Cypher) Decrypt(msg format.Message) ([]byte, error) {
+ filePart, err := ftCrypto.DecryptPart(
+ *c.key, msg.GetContents(), msg.GetMac(), c.fpNum, msg.GetKeyFP())
+ if err != nil {
+ return nil, err
+ }
+
+ c.fpVector.Use(uint32(c.fpNum))
+
+ return filePart, nil
+}
+
+// GetFingerprint generates and returns the fingerprints.
+func (c Cypher) GetFingerprint() format.Fingerprint {
+ return ftCrypto.GenerateFingerprint(*c.key, c.fpNum)
+}
diff --git a/fileTransfer2/store/cypher/cypher_test.go b/fileTransfer2/store/cypher/cypher_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..7fea485da2f35952ccdfea586aaa3aa7e5ab6d82
--- /dev/null
+++ b/fileTransfer2/store/cypher/cypher_test.go
@@ -0,0 +1,104 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package cypher
+
+import (
+ "bytes"
+ "gitlab.com/elixxir/primitives/format"
+ "testing"
+)
+
+// Tests that contents that are encrypted with Cypher.Encrypt match the
+// decrypted contents of Cypher.Decrypt.
+func TestCypher_Encrypt_Decrypt(t *testing.T) {
+ m, _ := newTestManager(16, t)
+ numPrimeBytes := 512
+
+ // Create contents of the right size
+ contents := make([]byte, format.NewMessage(numPrimeBytes).ContentsSize())
+ copy(contents, "This is some message contents.")
+
+ c, err := m.PopCypher()
+ if err != nil {
+ t.Errorf("Failed to pop cypher: %+v", err)
+ }
+
+ // Encrypt contents
+ cipherText, mac, fp := c.Encrypt(contents)
+
+ // Create message to decrypt
+ msg := format.NewMessage(numPrimeBytes)
+ msg.SetContents(cipherText)
+ msg.SetMac(mac)
+ msg.SetKeyFP(fp)
+
+ // Decrypt message
+ decryptedContents, err := c.Decrypt(msg)
+ if err != nil {
+ t.Errorf("Decrypt returned an error: %+v", err)
+ }
+
+ // Tests that the decrypted contents match the original
+ if !bytes.Equal(contents, decryptedContents) {
+ t.Errorf("Decrypted contents do not match original."+
+ "\nexpected: %q\nreceived: %q", contents, decryptedContents)
+ }
+}
+
+// Tests that Cypher.Decrypt returns an error when the contents are the wrong
+// size.
+func TestCypher_Decrypt_MacError(t *testing.T) {
+ m, _ := newTestManager(16, t)
+
+ // Create contents of the wrong size
+ contents := []byte("This is some message contents.")
+
+ c, err := m.PopCypher()
+ if err != nil {
+ t.Errorf("Failed to pop cypher: %+v", err)
+ }
+
+ // Encrypt contents
+ cipherText, mac, fp := c.Encrypt(contents)
+
+ // Create message to decrypt
+ msg := format.NewMessage(512)
+ msg.SetContents(cipherText)
+ msg.SetMac(mac)
+ msg.SetKeyFP(fp)
+
+ // Decrypt message
+ _, err = c.Decrypt(msg)
+ if err == nil {
+ t.Error("Failed to receive an error when the contents are the wrong " +
+ "length.")
+ }
+}
+
+// Tests that Cypher.GetFingerprint returns unique fingerprints.
+func TestCypher_GetFingerprint(t *testing.T) {
+ m, _ := newTestManager(16, t)
+ fpMap := make(map[format.Fingerprint]bool, m.fpVector.GetNumKeys())
+
+ for c, err := m.PopCypher(); err == nil; c, err = m.PopCypher() {
+ fp := c.GetFingerprint()
+
+ if fpMap[fp] {
+ t.Errorf("Fingerprint %s already exists.", fp)
+ } else {
+ fpMap[fp] = true
+ }
+ }
+}
diff --git a/fileTransfer2/store/cypher/manager.go b/fileTransfer2/store/cypher/manager.go
new file mode 100644
index 0000000000000000000000000000000000000000..f66100d4c27a608133f81424a32351fc1ba8bb58
--- /dev/null
+++ b/fileTransfer2/store/cypher/manager.go
@@ -0,0 +1,186 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package cypher
+
+import (
+ "github.com/pkg/errors"
+ "gitlab.com/elixxir/client/storage/utility"
+ "gitlab.com/elixxir/client/storage/versioned"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/xx_network/primitives/netTime"
+)
+
+// Storage keys and versions.
+const (
+ cypherManagerPrefix = "CypherManagerStore"
+ cypherManagerFpVectorKey = "CypherManagerFingerprintVector"
+ cypherManagerKeyStoreKey = "CypherManagerKey"
+ cypherManagerKeyStoreVersion = 0
+)
+
+// Error messages.
+const (
+ // NewManager
+ errNewFpVector = "failed to create new state vector for fingerprints: %+v"
+ errSaveKey = "failed to save transfer key: %+v"
+
+ // LoadManager
+ errLoadKey = "failed to load transfer key: %+v"
+ errLoadFpVector = "failed to load state vector: %+v"
+
+ // Manager.PopCypher
+ errGetNextFp = "used all %d fingerprints"
+
+ // Manager.Delete
+ errDeleteKey = "failed to delete transfer key: %+v"
+ errDeleteFpVector = "failed to delete fingerprint state vector: %+v"
+)
+
+// Manager the creation
+type Manager struct {
+ // The transfer key is a randomly generated key created by the sender and
+ // used to generate MACs and fingerprints
+ key *ftCrypto.TransferKey
+
+ // Stores the state of a fingerprint (used/unused) in a bitstream format
+ // (has its own storage backend)
+ fpVector *utility.StateVector
+
+ kv *versioned.KV
+}
+
+// NewManager returns a new cypher Manager initialised with the given number of
+// fingerprints.
+func NewManager(key *ftCrypto.TransferKey, numFps uint16, kv *versioned.KV) (
+ *Manager, error) {
+
+ kv = kv.Prefix(cypherManagerPrefix)
+
+ fpVector, err := utility.NewStateVector(
+ kv, cypherManagerFpVectorKey, uint32(numFps))
+ if err != nil {
+ return nil, errors.Errorf(errNewFpVector, err)
+ }
+
+ err = saveKey(key, kv)
+ if err != nil {
+ return nil, errors.Errorf(errSaveKey, err)
+ }
+
+ tfp := &Manager{
+ key: key,
+ fpVector: fpVector,
+ kv: kv,
+ }
+
+ return tfp, nil
+}
+
+// PopCypher returns a new Cypher with next available fingerprint number. This
+// marks the fingerprint as used. Returns false if no more fingerprints are
+// available.
+func (m *Manager) PopCypher() (Cypher, error) {
+ fpNum, err := m.fpVector.Next()
+ if err != nil {
+ return Cypher{}, errors.Errorf(errGetNextFp, m.fpVector.GetNumKeys())
+ }
+
+ c := Cypher{
+ Manager: m,
+ fpNum: uint16(fpNum),
+ }
+
+ return c, nil
+}
+
+// GetUnusedCyphers returns a list of cyphers with unused fingerprints numbers.
+func (m *Manager) GetUnusedCyphers() []Cypher {
+ fpNums := m.fpVector.GetUnusedKeyNums()
+ cypherList := make([]Cypher, len(fpNums))
+
+ for i, fpNum := range fpNums {
+ cypherList[i] = Cypher{
+ Manager: m,
+ fpNum: uint16(fpNum),
+ }
+ }
+
+ return cypherList
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Storage Functions //
+////////////////////////////////////////////////////////////////////////////////
+
+// LoadManager loads the Manager from storage.
+func LoadManager(kv *versioned.KV) (*Manager, error) {
+ kv = kv.Prefix(cypherManagerPrefix)
+ key, err := loadKey(kv)
+ if err != nil {
+ return nil, errors.Errorf(errLoadKey, err)
+ }
+
+ fpVector, err := utility.LoadStateVector(kv, cypherManagerFpVectorKey)
+ if err != nil {
+ return nil, errors.Errorf(errLoadFpVector, err)
+ }
+
+ tfp := &Manager{
+ key: key,
+ fpVector: fpVector,
+ kv: kv,
+ }
+
+ return tfp, nil
+}
+
+// Delete removes all saved entries from storage.
+func (m *Manager) Delete() error {
+ // Delete transfer key
+ err := m.kv.Delete(cypherManagerKeyStoreKey, cypherManagerKeyStoreVersion)
+ if err != nil {
+ return errors.Errorf(errDeleteKey, err)
+ }
+
+ // Delete StateVector
+ err = m.fpVector.Delete()
+ if err != nil {
+ return errors.Errorf(errDeleteFpVector, err)
+ }
+
+ return nil
+}
+
+// saveKey saves the transfer key to storage.
+func saveKey(key *ftCrypto.TransferKey, kv *versioned.KV) error {
+ obj := &versioned.Object{
+ Version: cypherManagerKeyStoreVersion,
+ Timestamp: netTime.Now(),
+ Data: key.Bytes(),
+ }
+
+ return kv.Set(cypherManagerKeyStoreKey, cypherManagerKeyStoreVersion, obj)
+}
+
+// loadKey loads the transfer key from storage.
+func loadKey(kv *versioned.KV) (*ftCrypto.TransferKey, error) {
+ obj, err := kv.Get(cypherManagerKeyStoreKey, cypherManagerKeyStoreVersion)
+ if err != nil {
+ return nil, err
+ }
+
+ key := ftCrypto.UnmarshalTransferKey(obj.Data)
+ return &key, nil
+}
diff --git a/fileTransfer2/store/cypher/manager_test.go b/fileTransfer2/store/cypher/manager_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..d354adcd6aed068e8c182102d7a7161060415ec9
--- /dev/null
+++ b/fileTransfer2/store/cypher/manager_test.go
@@ -0,0 +1,177 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package cypher
+
+import (
+ "fmt"
+ "gitlab.com/elixxir/client/storage/utility"
+ "gitlab.com/elixxir/client/storage/versioned"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/elixxir/ekv"
+ "gitlab.com/xx_network/crypto/csprng"
+ "reflect"
+ "testing"
+)
+
+// Tests that NewManager returns a new Manager that matches the expected
+// manager.
+func TestNewManager(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ numFps := uint16(64)
+ fpv, _ := utility.NewStateVector(kv.Prefix(cypherManagerPrefix),
+ cypherManagerFpVectorKey, uint32(numFps))
+ expected := &Manager{
+ key: &ftCrypto.TransferKey{1, 2, 3},
+ fpVector: fpv,
+ kv: kv.Prefix(cypherManagerPrefix),
+ }
+
+ manager, err := NewManager(expected.key, numFps, kv)
+ if err != nil {
+ t.Errorf("NewManager returned an error: %+v", err)
+ }
+
+ if !reflect.DeepEqual(expected, manager) {
+ t.Errorf("New manager does not match expected."+
+ "\nexpected: %+v\nreceived: %+v", expected, manager)
+ }
+}
+
+// Tests that Manager.PopCypher returns cyphers with correct fingerprint numbers
+// and that trying to pop after the last pop results in an error.
+func TestManager_PopCypher(t *testing.T) {
+ m, _ := newTestManager(64, t)
+
+ for i := uint16(0); i < uint16(m.fpVector.GetNumKeys()); i++ {
+ c, err := m.PopCypher()
+ if err != nil {
+ t.Errorf("Failed to pop cypher #%d: %+v", i, err)
+ }
+
+ if c.fpNum != i {
+ t.Errorf("Fingerprint number does not match expected."+
+ "\nexpected: %d\nreceived: %d", i, c.fpNum)
+ }
+
+ if c.Manager != m {
+ t.Errorf("Cypher has wrong manager.\nexpected: %v\nreceived: %v",
+ m, c.Manager)
+ }
+ }
+
+ // Test that an error is returned when popping a cypher after all
+ // fingerprints have been used
+ expectedErr := fmt.Sprintf(errGetNextFp, m.fpVector.GetNumKeys())
+ _, err := m.PopCypher()
+ if err == nil || (err.Error() != expectedErr) {
+ t.Errorf("PopCypher did not return the expected error when all "+
+ "fingerprints should be used.\nexpected: %s\nreceived: %+v",
+ expectedErr, err)
+ }
+}
+
+// Tests Manager.GetUnusedCyphers
+func TestManager_GetUnusedCyphers(t *testing.T) {
+ m, _ := newTestManager(64, t)
+
+ // Use every other key
+ for i := uint32(0); i < m.fpVector.GetNumKeys(); i += 2 {
+ m.fpVector.Use(i)
+ }
+
+ // Check that every other key is in the list
+ for i, c := range m.GetUnusedCyphers() {
+ if c.fpNum != uint16(2*i)+1 {
+ t.Errorf("Fingerprint number #%d incorrect."+
+ "\nexpected: %d\nreceived: %d", i, 2*i+1, c.fpNum)
+ }
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Storage Functions //
+////////////////////////////////////////////////////////////////////////////////
+
+// Tests that a Manager loaded via LoadManager matches the original.
+func TestLoadManager(t *testing.T) {
+ m, kv := newTestManager(64, t)
+
+ // Use every other key
+ for i := uint32(0); i < m.fpVector.GetNumKeys(); i += 2 {
+ m.fpVector.Use(i)
+ }
+
+ newManager, err := LoadManager(kv)
+ if err != nil {
+ t.Errorf("Failed to load manager: %+v", err)
+ }
+
+ if !reflect.DeepEqual(m, newManager) {
+ t.Errorf("Loaded manager does not match original."+
+ "\nexpected: %+v\nreceived: %+v", m, newManager)
+ }
+}
+
+// Tests that Manager.Delete deletes the storage by trying to load the manager.
+func TestManager_Delete(t *testing.T) {
+ m, _ := newTestManager(64, t)
+
+ err := m.Delete()
+ if err != nil {
+ t.Errorf("Failed to delete manager: %+v", err)
+ }
+
+ _, err = LoadManager(m.kv)
+ if err == nil {
+ t.Error("Failed to receive error when loading manager that was deleted.")
+ }
+}
+
+// Tests that a transfer key saved via saveKey can be loaded via loadKey.
+func Test_saveKey_loadKey(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ key := &ftCrypto.TransferKey{42}
+
+ err := saveKey(key, kv)
+ if err != nil {
+ t.Errorf("Error when saving key: %+v", err)
+ }
+
+ loadedKey, err := loadKey(kv)
+ if err != nil {
+ t.Errorf("Error when loading key: %+v", err)
+ }
+
+ if *key != *loadedKey {
+ t.Errorf("Loaded key does not match original."+
+ "\nexpected: %s\nreceived: %s", key, loadedKey)
+ }
+}
+
+// newTestManager creates a new Manager for testing.
+func newTestManager(numFps uint16, t *testing.T) (*Manager, *versioned.KV) {
+ key, err := ftCrypto.NewTransferKey(csprng.NewSystemRNG())
+ if err != nil {
+ t.Errorf("Failed to generate transfer key: %+v", err)
+ }
+
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ m, err := NewManager(&key, numFps, kv)
+ if err != nil {
+ t.Errorf("Failed to make new Manager: %+v", err)
+ }
+
+ return m, kv
+}
diff --git a/fileTransfer2/store/fileMessage/fileMessage.go b/fileTransfer2/store/fileMessage/fileMessage.go
new file mode 100644
index 0000000000000000000000000000000000000000..58f3b1273de9812be5960c95eac06f649d23b0e8
--- /dev/null
+++ b/fileTransfer2/store/fileMessage/fileMessage.go
@@ -0,0 +1,124 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileMessage
+
+import (
+ "encoding/binary"
+ "github.com/pkg/errors"
+ jww "github.com/spf13/jwalterweatherman"
+)
+
+// Size constants.
+const (
+ partNumLen = 2 // The length of the part number in bytes
+ fmMinSize = partNumLen // Minimum size for the PartMessage
+)
+
+// Error messages.
+const (
+ errNewFmSize = "[FT] Could not create file part message: size of payload (%d) must be greater than %d"
+ unmarshalFmSizeErr = "size of passed in bytes (%d) must be greater than %d"
+ errSetFileFm = "[FT] Could not set file part message payload: length of part bytes (%d) must be smaller than maximum payload size %d"
+)
+
+/*
++-------------------------------+
+| CMIX Message Contents |
++-------------+-----------------+
+| Part Number | File Data |
+| 2 bytes | remaining space |
++-------------+-----------------+
+*/
+
+// PartMessage contains part of the data being transferred and 256-bit nonce
+// that is used as a nonce.
+type PartMessage struct {
+ data []byte // Serial of all contents
+ partNum []byte // The part number of the file
+ part []byte // File part data
+}
+
+// NewPartMessage generates a new part message that fits into the specified
+// external payload size. An error is returned if the external payload size is
+// too small to fit the part message.
+func NewPartMessage(externalPayloadSize int) PartMessage {
+ if externalPayloadSize < fmMinSize {
+ jww.FATAL.Panicf(errNewFmSize, externalPayloadSize, fmMinSize)
+ }
+
+ return mapPartMessage(make([]byte, externalPayloadSize))
+}
+
+// mapPartMessage maps the data to the components of a PartMessage. It is mapped
+// by reference; a copy is not made.
+func mapPartMessage(data []byte) PartMessage {
+ return PartMessage{
+ data: data,
+ partNum: data[:partNumLen],
+ part: data[partNumLen:],
+ }
+}
+
+// UnmarshalPartMessage converts the bytes into a PartMessage. An error is
+// returned if the size of the data is too small for a PartMessage.
+func UnmarshalPartMessage(b []byte) (PartMessage, error) {
+ if len(b) < fmMinSize {
+ return PartMessage{},
+ errors.Errorf(unmarshalFmSizeErr, len(b), fmMinSize)
+ }
+
+ return mapPartMessage(b), nil
+}
+
+// Marshal returns the byte representation of the PartMessage.
+func (m PartMessage) Marshal() []byte {
+ b := make([]byte, len(m.data))
+ copy(b, m.data)
+ return b
+}
+
+// GetPartNum returns the file part number.
+func (m PartMessage) GetPartNum() uint16 {
+ return binary.LittleEndian.Uint16(m.partNum)
+}
+
+// SetPartNum sets the file part number.
+func (m PartMessage) SetPartNum(num uint16) {
+ b := make([]byte, partNumLen)
+ binary.LittleEndian.PutUint16(b, num)
+ copy(m.partNum, b)
+}
+
+// GetPart returns the file part data from the message.
+func (m PartMessage) GetPart() []byte {
+ b := make([]byte, len(m.part))
+ copy(b, m.part)
+ return b
+}
+
+// SetPart sets the PartMessage part to the given bytes. An error is returned if
+// the size of the provided part data is too large to store.
+func (m PartMessage) SetPart(b []byte) {
+ if len(b) > len(m.part) {
+ jww.FATAL.Panicf(errSetFileFm, len(b), len(m.part))
+ }
+
+ copy(m.part, b)
+}
+
+// GetPartSize returns the number of bytes available to store part data.
+func (m PartMessage) GetPartSize() int {
+ return len(m.part)
+}
diff --git a/fileTransfer2/store/fileMessage/fileMessage_test.go b/fileTransfer2/store/fileMessage/fileMessage_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..f7e169456e1d9dee3121e7a653b9b109417036a9
--- /dev/null
+++ b/fileTransfer2/store/fileMessage/fileMessage_test.go
@@ -0,0 +1,236 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileMessage
+
+import (
+ "bytes"
+ "encoding/binary"
+ "fmt"
+ "math/rand"
+ "testing"
+)
+
+// Tests that NewPartMessage returns a PartMessage of the expected size.
+func Test_newPartMessage(t *testing.T) {
+ externalPayloadSize := 256
+
+ fm := NewPartMessage(externalPayloadSize)
+
+ if len(fm.data) != externalPayloadSize {
+ t.Errorf("Size of PartMessage data does not match payload size."+
+ "\nexpected: %d\nreceived: %d", externalPayloadSize, len(fm.data))
+ }
+}
+
+// Error path: tests that NewPartMessage returns the expected error when the
+// external payload size is too small.
+func Test_newPartMessage_SmallPayloadSizeError(t *testing.T) {
+ externalPayloadSize := fmMinSize - 1
+ expectedErr := fmt.Sprintf(errNewFmSize, externalPayloadSize, fmMinSize)
+
+ defer func() {
+ if r := recover(); r == nil || r != expectedErr {
+ t.Errorf("NewPartMessage did not return the expected error when "+
+ "the given external payload size is too small."+
+ "\nexpected: %s\nreceived: %+v", expectedErr, r)
+ }
+ }()
+
+ NewPartMessage(externalPayloadSize)
+}
+
+// Tests that mapPartMessage maps the data to the correct parts of the
+// PartMessage.
+func Test_mapPartMessage(t *testing.T) {
+ // Generate expected values
+ _, expectedData, expectedPartNum, expectedFile :=
+ newRandomFileMessage()
+
+ fm := mapPartMessage(expectedData)
+
+ if !bytes.Equal(expectedData, fm.data) {
+ t.Errorf("Incorrect data.\nexpected: %q\nreceived: %q",
+ expectedData, fm.data)
+ }
+
+ if !bytes.Equal(expectedPartNum, fm.partNum) {
+ t.Errorf("Incorrect part number.\nexpected: %q\nreceived: %q",
+ expectedPartNum, fm.partNum)
+ }
+
+ if !bytes.Equal(expectedFile, fm.part) {
+ t.Errorf("Incorrect part data.\nexpected: %q\nreceived: %q",
+ expectedFile, fm.part)
+ }
+
+}
+
+// Tests that UnmarshalPartMessage returns a PartMessage with the expected
+// values.
+func Test_unmarshalPartMessage(t *testing.T) {
+ // Generate expected values
+ _, expectedData, expectedPartNumb, expectedFile :=
+ newRandomFileMessage()
+
+ fm, err := UnmarshalPartMessage(expectedData)
+ if err != nil {
+ t.Errorf("UnmarshalPartMessage return an error: %+v", err)
+ }
+
+ if !bytes.Equal(expectedData, fm.data) {
+ t.Errorf("Incorrect data.\nexpected: %q\nreceived: %q",
+ expectedData, fm.data)
+ }
+
+ if !bytes.Equal(expectedPartNumb, fm.partNum) {
+ t.Errorf("Incorrect part number.\nexpected: %q\nreceived: %q",
+ expectedPartNumb, fm.partNum)
+ }
+
+ if !bytes.Equal(expectedFile, fm.part) {
+ t.Errorf("Incorrect part data.\nexpected: %q\nreceived: %q",
+ expectedFile, fm.part)
+ }
+}
+
+// Error path: tests that UnmarshalPartMessage returns the expected error when
+// the provided data is too small to be unmarshalled into a PartMessage.
+func Test_unmarshalPartMessage_SizeError(t *testing.T) {
+ data := make([]byte, fmMinSize-1)
+ expectedErr := fmt.Sprintf(unmarshalFmSizeErr, len(data), fmMinSize)
+
+ _, err := UnmarshalPartMessage(data)
+ if err == nil || err.Error() != expectedErr {
+ t.Errorf("UnmarshalPartMessage did not return the expected error when "+
+ "the given bytes are too small to be a PartMessage."+
+ "\nexpected: %s\nreceived: %+v", expectedErr, err)
+ }
+}
+
+// Tests that PartMessage.Marshal returns the correct data.
+func Test_fileMessage_marshal(t *testing.T) {
+ fm, expectedData, _, _ := newRandomFileMessage()
+
+ data := fm.Marshal()
+
+ if !bytes.Equal(expectedData, data) {
+ t.Errorf("Marshalled data does not match expected."+
+ "\nexpected: %q\nreceived: %q", expectedData, data)
+ }
+}
+
+// Tests that PartMessage.GetPartNum returns the correct part number.
+func Test_fileMessage_getPartNum(t *testing.T) {
+ fm, _, expectedPartNum, _ := newRandomFileMessage()
+
+ partNum := fm.GetPartNum()
+ expected := binary.LittleEndian.Uint16(expectedPartNum)
+
+ if expected != partNum {
+ t.Errorf("Part number does not match expected."+
+ "\nexpected: %d\nreceived: %d", expected, partNum)
+ }
+}
+
+// Tests that PartMessage.SetPartNum sets the correct part number.
+func Test_fileMessage_setPartNum(t *testing.T) {
+ fm := NewPartMessage(256)
+
+ expectedPartNum := make([]byte, partNumLen)
+ rand.New(rand.NewSource(42)).Read(expectedPartNum)
+ expected := binary.LittleEndian.Uint16(expectedPartNum)
+
+ fm.SetPartNum(expected)
+
+ if expected != fm.GetPartNum() {
+ t.Errorf("Failed to set correct part number.\nexpected: %d\nreceived: %d",
+ expected, fm.GetPartNum())
+ }
+}
+
+// Tests that PartMessage.GetPart returns the correct part data.
+func Test_fileMessage_getFile(t *testing.T) {
+ fm, _, _, expectedFile := newRandomFileMessage()
+
+ file := fm.GetPart()
+
+ if !bytes.Equal(expectedFile, file) {
+ t.Errorf("File data does not match expected."+
+ "\nexpected: %q\nreceived: %q", expectedFile, file)
+ }
+}
+
+// Tests that PartMessage.SetPart sets the correct part data.
+func Test_fileMessage_setFile(t *testing.T) {
+ fm := NewPartMessage(256)
+
+ fileData := make([]byte, 64)
+ rand.New(rand.NewSource(42)).Read(fileData)
+ expectedFile := make([]byte, fm.GetPartSize())
+ copy(expectedFile, fileData)
+
+ fm.SetPart(expectedFile)
+
+ if !bytes.Equal(expectedFile, fm.GetPart()) {
+ t.Errorf("Failed to set correct part data.\nexpected: %q\nreceived: %q",
+ expectedFile, fm.GetPart())
+ }
+}
+
+// Error path: tests that PartMessage.SetPart returns the expected error when
+// the provided part data is too large for the message.
+func Test_fileMessage_setFile_FileTooLargeError(t *testing.T) {
+ fm := NewPartMessage(fmMinSize + 1)
+
+ expectedErr := fmt.Sprintf(errSetFileFm, fm.GetPartSize()+1, fm.GetPartSize())
+
+ defer func() {
+ if r := recover(); r == nil || r != expectedErr {
+ t.Errorf("SetPart did not return the expected error when the "+
+ "given part data is too large to fit in the PartMessage."+
+ "\nexpected: %s\nreceived: %+v", expectedErr, r)
+ }
+ }()
+
+ fm.SetPart(make([]byte, fm.GetPartSize()+1))
+}
+
+// Tests that PartMessage.GetPartSize returns the expected available space for
+// the part data.
+func Test_fileMessage_getFileSize(t *testing.T) {
+ expectedSize := 256
+
+ fm := NewPartMessage(fmMinSize + expectedSize)
+
+ if expectedSize != fm.GetPartSize() {
+ t.Errorf("File size incorrect.\nexpected: %d\nreceived: %d",
+ expectedSize, fm.GetPartSize())
+ }
+}
+
+// newRandomFileMessage generates a new PartMessage filled with random data and
+// return the PartMessage and its individual parts.
+func newRandomFileMessage() (PartMessage, []byte, []byte, []byte) {
+ prng := rand.New(rand.NewSource(42))
+ partNum := make([]byte, partNumLen)
+ prng.Read(partNum)
+ part := make([]byte, 64)
+ prng.Read(part)
+ data := append(partNum, part...)
+
+ fm := mapPartMessage(data)
+
+ return fm, data, partNum, part
+}
diff --git a/fileTransfer2/store/part.go b/fileTransfer2/store/part.go
new file mode 100644
index 0000000000000000000000000000000000000000..9229cfb008a8da189a7fd00d401cae8a31ed3c5a
--- /dev/null
+++ b/fileTransfer2/store/part.go
@@ -0,0 +1,70 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package store
+
+import (
+ "gitlab.com/elixxir/client/fileTransfer2/store/cypher"
+ "gitlab.com/elixxir/client/fileTransfer2/store/fileMessage"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/elixxir/primitives/format"
+ "gitlab.com/xx_network/primitives/id"
+)
+
+// Part contains information about a single file part and its parent transfer.
+// Also contains cryptographic information needed to encrypt the part data.
+type Part struct {
+ transfer *SentTransfer
+ cypherManager *cypher.Manager
+ partNum uint16
+}
+
+// GetEncryptedPart gets the specified part, encrypts it, and returns the
+// encrypted part along with its MAC and fingerprint. An error is returned if no
+// fingerprints are available.
+func (p *Part) GetEncryptedPart(contentsSize int) (
+ encryptedPart, mac []byte, fp format.Fingerprint, err error) {
+ // Create new empty file part message of the size provided
+ partMsg := fileMessage.NewPartMessage(contentsSize)
+
+ // Add part number and part data to part message
+ partMsg.SetPartNum(p.partNum)
+ partMsg.SetPart(p.transfer.getPartData(p.partNum))
+
+ // Get next cypher
+ c, err := p.cypherManager.PopCypher()
+ if err != nil {
+ p.transfer.markTransferFailed()
+ return nil, nil, format.Fingerprint{}, err
+ }
+
+ // Encrypt part and get MAC and fingerprint
+ encryptedPart, mac, fp = c.Encrypt(partMsg.Marshal())
+
+ return encryptedPart, mac, fp, nil
+}
+
+// MarkArrived marks the part as arrived. This should be called after the round
+// the part is sent on succeeds.
+func (p *Part) MarkArrived() {
+ p.transfer.markArrived(p.partNum)
+}
+
+// Recipient returns the recipient of the file transfer.
+func (p *Part) Recipient() *id.ID {
+ return p.transfer.recipient
+}
+
+// TransferID returns the ID of the file transfer.
+func (p *Part) TransferID() *ftCrypto.TransferID {
+ return p.transfer.tid
+}
+
+// FileName returns the name of the file.
+func (p *Part) FileName() string {
+ return p.transfer.FileName()
+}
diff --git a/fileTransfer2/store/part_test.go b/fileTransfer2/store/part_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..67748160971e47c378c7c5fb31fb188014244fb7
--- /dev/null
+++ b/fileTransfer2/store/part_test.go
@@ -0,0 +1,126 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package store
+
+import (
+ "bytes"
+ "gitlab.com/elixxir/client/fileTransfer2/store/fileMessage"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/elixxir/primitives/format"
+ "testing"
+)
+
+// Tests that the encrypted part returned by Part.GetEncryptedPart can be
+// decrypted, unmarshalled, and that it matches the original.
+func TestPart_GetEncryptedPart(t *testing.T) {
+ st, parts, key, _, _ := newTestSentTransfer(25, t)
+ partNum := 0
+ part := st.GetUnsentParts()[partNum]
+
+ encryptedPart, mac, fp, err := part.GetEncryptedPart(
+ format.NewMessage(numPrimeBytes).ContentsSize())
+ if err != nil {
+ t.Errorf("GetEncryptedPart returned an error: %+v", err)
+ }
+
+ decryptedPart, err := ftCrypto.DecryptPart(
+ *key, encryptedPart, mac, uint16(partNum), fp)
+ if err != nil {
+ t.Errorf("Failed to decrypt part: %+v", err)
+ }
+
+ partMsg, err := fileMessage.UnmarshalPartMessage(decryptedPart)
+ if err != nil {
+ t.Errorf("Failed to unmarshal part message: %+v", err)
+ }
+
+ if !bytes.Equal(parts[partNum], partMsg.GetPart()) {
+ t.Errorf("Decrypted part does not match original."+
+ "\nexpected: %q\nreceived: %q", parts[partNum], partMsg.GetPart())
+ }
+
+ if int(partMsg.GetPartNum()) != partNum {
+ t.Errorf("Decrypted part does not have correct part number."+
+ "\nexpected: %d\nreceived: %d", partNum, partMsg.GetPartNum())
+ }
+}
+
+// Tests that Part.GetEncryptedPart returns an error when the underlying cypher
+// manager runs out of fingerprints.
+func TestPart_GetEncryptedPart_OutOfFingerprints(t *testing.T) {
+ numParts := uint16(25)
+ st, _, _, numFps, _ := newTestSentTransfer(numParts, t)
+ part := st.GetUnsentParts()[0]
+ for i := uint16(0); i < numFps; i++ {
+ _, _, _, err := part.GetEncryptedPart(
+ format.NewMessage(numPrimeBytes).ContentsSize())
+ if err != nil {
+ t.Errorf("Getting encrtypted part %d failed: %+v", i, err)
+ }
+ }
+
+ _, _, _, err := part.GetEncryptedPart(
+ format.NewMessage(numPrimeBytes).ContentsSize())
+ if err == nil {
+ t.Errorf("Failed to get an error when run out of fingerprints.")
+ }
+}
+
+// Tests that Part.MarkArrived correctly marks the part's status in the
+// SentTransfer's partStatus vector.
+func TestPart_MarkArrived(t *testing.T) {
+ st, _, _, _, _ := newTestSentTransfer(25, t)
+ partNum := 0
+ part := st.GetUnsentParts()[partNum]
+
+ part.MarkArrived()
+
+ if !st.partStatus.Used(uint32(partNum)) {
+ t.Errorf("Part #%d not marked as arrived.", partNum)
+ }
+}
+
+// Tests that Part.Recipient returns the correct recipient ID.
+func TestPart_Recipient(t *testing.T) {
+ st, _, _, _, _ := newTestSentTransfer(25, t)
+ part := st.GetUnsentParts()[0]
+
+ if !part.Recipient().Cmp(st.Recipient()) {
+ t.Errorf("Recipient ID does not match expected."+
+ "\nexpected: %s\nreceived: %s", st.Recipient(), part.Recipient())
+ }
+}
+
+// Tests that Part.TransferID returns the correct transfer ID.
+func TestPart_TransferID(t *testing.T) {
+ st, _, _, _, _ := newTestSentTransfer(25, t)
+ part := st.GetUnsentParts()[0]
+
+ if part.TransferID() != st.TransferID() {
+ t.Errorf("Transfer ID does not match expected."+
+ "\nexpected: %s\nreceived: %s", st.TransferID(), part.TransferID())
+ }
+}
+
+// Tests that Part.FileName returns the correct file name.
+func TestPart_FileName(t *testing.T) {
+ st, _, _, _, _ := newTestSentTransfer(25, t)
+ part := st.GetUnsentParts()[0]
+
+ if part.FileName() != st.FileName() {
+ t.Errorf("File name does not match expected."+
+ "\nexpected: %q\nreceived: %q", st.FileName(), part.FileName())
+ }
+}
diff --git a/fileTransfer2/store/received.go b/fileTransfer2/store/received.go
new file mode 100644
index 0000000000000000000000000000000000000000..81c9439217d06679fe17d9e41968575526780adf
--- /dev/null
+++ b/fileTransfer2/store/received.go
@@ -0,0 +1,181 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package store
+
+import (
+ "encoding/json"
+ "github.com/pkg/errors"
+ jww "github.com/spf13/jwalterweatherman"
+ "gitlab.com/elixxir/client/storage/versioned"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/elixxir/ekv"
+ "gitlab.com/xx_network/primitives/netTime"
+ "sync"
+)
+
+// Storage keys and versions.
+const (
+ receivedTransfersStorePrefix = "ReceivedFileTransfersPrefix"
+ receivedTransfersStoreKey = "ReceivedFileTransfers"
+ receivedTransfersStoreVersion = 0
+)
+
+// Error messages.
+const (
+ // NewOrLoadReceived
+ errLoadReceived = "error loading received transfer list from storage: %+v"
+ errUnmarshalReceived = "could not unmarshal received transfer list: %+v"
+ warnLoadReceivedTransfer = "[FT] failed to load received transfer %d of %d with ID %s: %+v"
+ errLoadAllReceivedTransfer = "failed to load all %d transfers"
+
+ // Received.AddTransfer
+ errAddExistingReceivedTransfer = "received transfer with ID %s already exists in map."
+)
+
+// Received contains a list of all received transfers.
+type Received struct {
+ transfers map[ftCrypto.TransferID]*ReceivedTransfer
+
+ mux sync.RWMutex
+ kv *versioned.KV
+}
+
+// NewOrLoadReceived attempts to load a Received from storage. Or if none exist,
+// then a new Received is returned. Also returns a list of all transfers that
+// have unreceived file parts so their fingerprints can be re-added.
+func NewOrLoadReceived(kv *versioned.KV) (*Received, []*ReceivedTransfer, error) {
+ s := &Received{
+ transfers: make(map[ftCrypto.TransferID]*ReceivedTransfer),
+ kv: kv.Prefix(receivedTransfersStorePrefix),
+ }
+
+ obj, err := s.kv.Get(receivedTransfersStoreKey, receivedTransfersStoreVersion)
+ if err != nil {
+ if ekv.Exists(err) {
+ return nil, nil, errors.Errorf(errLoadReceived, err)
+ } else {
+ return s, nil, nil
+ }
+ }
+
+ tidList, err := unmarshalTransferIdList(obj.Data)
+ if err != nil {
+ return nil, nil, errors.Errorf(errUnmarshalReceived, err)
+ }
+
+ var errCount int
+ unfinishedTransfer := make([]*ReceivedTransfer, 0, len(tidList))
+ for i := range tidList {
+ tid := tidList[i]
+ s.transfers[tid], err = loadReceivedTransfer(&tid, s.kv)
+ if err != nil {
+ jww.WARN.Print(warnLoadReceivedTransfer, i, len(tidList), tid, err)
+ errCount++
+ }
+
+ if s.transfers[tid].NumReceived() != s.transfers[tid].NumParts() {
+ unfinishedTransfer = append(unfinishedTransfer, s.transfers[tid])
+ }
+ }
+
+ // Return an error if all transfers failed to load
+ if errCount == len(tidList) {
+ return nil, nil, errors.Errorf(errLoadAllReceivedTransfer, len(tidList))
+ }
+
+ return s, unfinishedTransfer, nil
+}
+
+// AddTransfer adds the ReceivedTransfer to the map keyed on its transfer ID.
+func (r *Received) AddTransfer(key *ftCrypto.TransferKey,
+ tid *ftCrypto.TransferID, fileName string, transferMAC []byte, numParts,
+ numFps uint16, fileSize uint32) (*ReceivedTransfer, error) {
+
+ r.mux.Lock()
+ defer r.mux.Unlock()
+
+ _, exists := r.transfers[*tid]
+ if exists {
+ return nil, errors.Errorf(errAddExistingReceivedTransfer, tid)
+ }
+
+ rt, err := newReceivedTransfer(key, tid, fileName, transferMAC, numParts,
+ numFps, fileSize, r.kv)
+ if err != nil {
+ return nil, err
+ }
+
+ r.transfers[*tid] = rt
+
+ return rt, r.save()
+}
+
+// GetTransfer returns the ReceivedTransfer with the desiccated transfer ID or
+// false if none exists.
+func (r *Received) GetTransfer(tid *ftCrypto.TransferID) (*ReceivedTransfer, bool) {
+ r.mux.RLock()
+ defer r.mux.RUnlock()
+
+ rt, exists := r.transfers[*tid]
+ return rt, exists
+}
+
+// RemoveTransfer removes the transfer from the map. If no transfer exists,
+// returns nil. Only errors due to saving to storage are returned.
+func (r *Received) RemoveTransfer(tid *ftCrypto.TransferID) error {
+ r.mux.Lock()
+ defer r.mux.Unlock()
+
+ _, exists := r.transfers[*tid]
+ if !exists {
+ return nil
+ }
+
+ delete(r.transfers, *tid)
+ return r.save()
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Storage Functions //
+////////////////////////////////////////////////////////////////////////////////
+
+// save stores a list of transfer IDs in the map to storage.
+func (r *Received) save() error {
+ data, err := marshalReceivedTransfersMap(r.transfers)
+ if err != nil {
+ return err
+ }
+
+ obj := &versioned.Object{
+ Version: receivedTransfersStoreVersion,
+ Timestamp: netTime.Now(),
+ Data: data,
+ }
+
+ return r.kv.Set(receivedTransfersStoreKey, receivedTransfersStoreVersion, obj)
+}
+
+// marshalReceivedTransfersMap serialises the list of transfer IDs from a
+// ReceivedTransfer map.
+func marshalReceivedTransfersMap(
+ transfers map[ftCrypto.TransferID]*ReceivedTransfer) ([]byte, error) {
+ tidList := make([]ftCrypto.TransferID, 0, len(transfers))
+
+ for tid := range transfers {
+ tidList = append(tidList, tid)
+ }
+
+ return json.Marshal(tidList)
+}
diff --git a/fileTransfer2/store/receivedTransfer.go b/fileTransfer2/store/receivedTransfer.go
new file mode 100644
index 0000000000000000000000000000000000000000..e2ce440a0456542ae5455166130a2d239ed5d44a
--- /dev/null
+++ b/fileTransfer2/store/receivedTransfer.go
@@ -0,0 +1,377 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package store
+
+import (
+ "bytes"
+ "encoding/base64"
+ "encoding/json"
+ "github.com/pkg/errors"
+ jww "github.com/spf13/jwalterweatherman"
+ "gitlab.com/elixxir/client/fileTransfer2/store/cypher"
+ "gitlab.com/elixxir/client/storage/utility"
+ "gitlab.com/elixxir/client/storage/versioned"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/xx_network/primitives/netTime"
+ "strconv"
+ "sync"
+)
+
+// Storage keys and versions.
+const (
+ receivedTransferStorePrefix = "ReceivedFileTransferStore/"
+ receivedTransferStoreKey = "ReceivedTransfer"
+ receivedTransferStoreVersion = 0
+ receivedTransferStatusKey = "ReceivedPartStatusVector"
+ receivedPartStoreKey = "receivedPart#"
+ receivedPartStoreVersion = 0
+)
+
+// Error messages.
+const (
+ // newReceivedTransfer
+ errRtNewCypherManager = "failed to create new cypher manager: %+v"
+ errRtNewPartStatusVectorErr = "failed to create new state vector for part statuses: %+v"
+
+ // ReceivedTransfer.AddPart
+ errPartOutOfRange = "part number %d out of range of max %d"
+ errReceivedPartSave = "failed to save part #%d to storage: %+v"
+
+ // loadReceivedTransfer
+ errRtLoadCypherManager = "failed to load cypher manager from storage: %+v"
+ errRtLoadFields = "failed to load transfer MAC, number of parts, and file size: %+v"
+ errRtUnmarshalFields = "failed to unmarshal transfer MAC, number of parts, and file size: %+v"
+ errRtLoadPartStatusVector = "failed to load state vector for part statuses: %+v"
+ errRtLoadPart = "[FT] Failed to load part #%d from storage: %+v"
+
+ // ReceivedTransfer.Delete
+ errRtDeleteCypherManager = "failed to delete cypher manager: %+v"
+ errRtDeleteSentTransfer = "failed to delete transfer MAC, number of parts, and file size: %+v"
+ errRtDeletePartStatus = "failed to delete part status state vector: %+v"
+
+ // ReceivedTransfer.save
+ errMarshalReceivedTransfer = "failed to marshal: %+v"
+)
+
+// ReceivedTransfer contains information and progress data for a receiving or
+// received file transfer.
+type ReceivedTransfer struct {
+ // Tracks file part cyphers
+ cypherManager *cypher.Manager
+
+ // The ID of the transfer
+ tid *ftCrypto.TransferID
+
+ // User given name to file
+ fileName string
+
+ // The MAC for the entire file; used to verify the integrity of all parts
+ transferMAC []byte
+
+ // The number of file parts in the file
+ numParts uint16
+
+ // Size of the entire file in bytes
+ fileSize uint32
+
+ // Saves each part in order (has its own storage backend)
+ parts [][]byte
+
+ // Stores the received status for each file part in a bitstream format
+ partStatus *utility.StateVector
+
+ mux sync.RWMutex
+ kv *versioned.KV
+}
+
+// newReceivedTransfer generates a ReceivedTransfer with the specified transfer
+// key, transfer ID, and a number of parts.
+func newReceivedTransfer(key *ftCrypto.TransferKey, tid *ftCrypto.TransferID,
+ fileName string, transferMAC []byte, numParts, numFps uint16,
+ fileSize uint32, kv *versioned.KV) (*ReceivedTransfer, error) {
+ kv = kv.Prefix(makeReceivedTransferPrefix(tid))
+
+ // Create new cypher manager
+ cypherManager, err := cypher.NewManager(key, numFps, kv)
+ if err != nil {
+ return nil, errors.Errorf(errRtNewCypherManager, err)
+ }
+
+ // Create new state vector for storing statuses of received parts
+ partStatus, err := utility.NewStateVector(
+ kv, receivedTransferStatusKey, uint32(numParts))
+ if err != nil {
+ return nil, errors.Errorf(errRtNewPartStatusVectorErr, err)
+ }
+
+ rt := &ReceivedTransfer{
+ cypherManager: cypherManager,
+ tid: tid,
+ fileName: fileName,
+ transferMAC: transferMAC,
+ numParts: numParts,
+ fileSize: fileSize,
+ parts: make([][]byte, numParts),
+ partStatus: partStatus,
+ kv: kv,
+ }
+
+ return rt, rt.save()
+}
+
+// AddPart adds the file part to the list of file parts at the index of partNum.
+func (rt *ReceivedTransfer) AddPart(part []byte, partNum int) error {
+ rt.mux.Lock()
+ defer rt.mux.Unlock()
+
+ if partNum > len(rt.parts)-1 {
+ return errors.Errorf(errPartOutOfRange, partNum, len(rt.parts)-1)
+ }
+
+ // Save part
+ rt.parts[partNum] = part
+ err := savePart(part, partNum, rt.kv)
+ if err != nil {
+ return errors.Errorf(errReceivedPartSave, partNum, err)
+ }
+
+ // Mark part as received
+ rt.partStatus.Use(uint32(partNum))
+
+ return nil
+}
+
+// GetFile concatenates all file parts and returns it as a single complete file.
+// Note that this function does not care for the completeness of the file and
+// returns all parts it has.
+func (rt *ReceivedTransfer) GetFile() []byte {
+ rt.mux.RLock()
+ defer rt.mux.RUnlock()
+
+ file := bytes.Join(rt.parts, nil)
+
+ // Strip off trailing padding from last part
+ if len(file) > int(rt.fileSize) {
+ file = file[:rt.fileSize]
+ }
+
+ return file
+}
+
+// GetUnusedCyphers returns a list of cyphers with unused fingerprint numbers.
+func (rt *ReceivedTransfer) GetUnusedCyphers() []cypher.Cypher {
+ return rt.cypherManager.GetUnusedCyphers()
+}
+
+// NumParts returns the total number of file parts in the transfer.
+func (rt *ReceivedTransfer) NumParts() uint16 {
+ return rt.numParts
+}
+
+// TransferID returns the transfer's ID.
+func (rt *ReceivedTransfer) TransferID() *ftCrypto.TransferID {
+ return rt.tid
+}
+
+// FileName returns the transfer's file name.
+func (rt *ReceivedTransfer) FileName() string {
+ return rt.fileName
+}
+
+// NumReceived returns the number of parts that have been received.
+func (rt *ReceivedTransfer) NumReceived() uint16 {
+ rt.mux.RLock()
+ defer rt.mux.RUnlock()
+ return uint16(rt.partStatus.GetNumUsed())
+}
+
+// CopyPartStatusVector returns a copy of the part status vector that can be
+// used to look up the current status of parts. Note that the statuses are from
+// when this function is called and not realtime.
+func (rt *ReceivedTransfer) CopyPartStatusVector() *utility.StateVector {
+ return rt.partStatus.DeepCopy()
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Storage Functions //
+////////////////////////////////////////////////////////////////////////////////
+
+// loadReceivedTransfer loads the ReceivedTransfer with the given transfer ID
+// from storage.
+func loadReceivedTransfer(tid *ftCrypto.TransferID, kv *versioned.KV) (
+ *ReceivedTransfer, error) {
+ kv = kv.Prefix(makeReceivedTransferPrefix(tid))
+
+ // Load cypher manager
+ cypherManager, err := cypher.LoadManager(kv)
+ if err != nil {
+ return nil, errors.Errorf(errRtLoadCypherManager, err)
+ }
+
+ // Load transfer MAC, number of parts, and file size
+ obj, err := kv.Get(receivedTransferStoreKey, receivedTransferStoreVersion)
+ if err != nil {
+ return nil, errors.Errorf(errRtLoadFields, err)
+ }
+
+ fileName, transferMAC, numParts, fileSize, err :=
+ unmarshalReceivedTransfer(obj.Data)
+ if err != nil {
+ return nil, errors.Errorf(errRtUnmarshalFields, err)
+ }
+
+ // Load StateVector for storing statuses of received parts
+ partStatus, err := utility.LoadStateVector(kv, receivedTransferStatusKey)
+ if err != nil {
+ return nil, errors.Errorf(errRtLoadPartStatusVector, err)
+ }
+
+ // Load parts from storage
+ parts := make([][]byte, numParts)
+ for i := range parts {
+ if partStatus.Used(uint32(i)) {
+ parts[i], err = loadPart(i, kv)
+ if err != nil {
+ jww.ERROR.Printf(errRtLoadPart, i, err)
+ }
+ }
+ }
+
+ rt := &ReceivedTransfer{
+ cypherManager: cypherManager,
+ tid: tid,
+ fileName: fileName,
+ transferMAC: transferMAC,
+ numParts: numParts,
+ fileSize: fileSize,
+ parts: parts,
+ partStatus: partStatus,
+ kv: kv,
+ }
+
+ return rt, nil
+}
+
+// Delete deletes all data in the ReceivedTransfer from storage.
+func (rt *ReceivedTransfer) Delete() error {
+ rt.mux.Lock()
+ defer rt.mux.Unlock()
+
+ // Delete cypher manager
+ err := rt.cypherManager.Delete()
+ if err != nil {
+ return errors.Errorf(errRtDeleteCypherManager, err)
+ }
+
+ // Delete transfer MAC, number of parts, and file size
+ err = rt.kv.Delete(receivedTransferStoreKey, receivedTransferStoreVersion)
+ if err != nil {
+ return errors.Errorf(errRtDeleteSentTransfer, err)
+ }
+
+ // Delete part status state vector
+ err = rt.partStatus.Delete()
+ if err != nil {
+ return errors.Errorf(errRtDeletePartStatus, err)
+ }
+
+ return nil
+}
+
+// save stores all fields in ReceivedTransfer that do not have their own storage
+// (transfer MAC, file size, and number of file parts) to storage.
+func (rt *ReceivedTransfer) save() error {
+ data, err := rt.marshal()
+ if err != nil {
+ return errors.Errorf(errMarshalReceivedTransfer, err)
+ }
+
+ // Create new versioned object for the ReceivedTransfer
+ vo := &versioned.Object{
+ Version: receivedTransferStoreVersion,
+ Timestamp: netTime.Now(),
+ Data: data,
+ }
+
+ // Save versioned object
+ return rt.kv.Set(receivedTransferStoreKey, receivedTransferStoreVersion, vo)
+}
+
+// receivedTransferDisk structure is used to marshal and unmarshal
+// ReceivedTransfer fields to/from storage.
+type receivedTransferDisk struct {
+ FileName string
+ TransferMAC []byte
+ NumParts uint16
+ FileSize uint32
+}
+
+// marshal serialises the ReceivedTransfer's fileName, transferMAC, numParts,
+// and fileSize.
+func (rt *ReceivedTransfer) marshal() ([]byte, error) {
+ disk := receivedTransferDisk{
+ FileName: rt.fileName,
+ TransferMAC: rt.transferMAC,
+ NumParts: rt.numParts,
+ FileSize: rt.fileSize,
+ }
+
+ return json.Marshal(disk)
+}
+
+// unmarshalReceivedTransfer deserializes the data into the fileName,
+// transferMAC, numParts, and fileSize.
+func unmarshalReceivedTransfer(data []byte) (fileName string,
+ transferMAC []byte, numParts uint16, fileSize uint32, err error) {
+ var disk receivedTransferDisk
+ err = json.Unmarshal(data, &disk)
+ if err != nil {
+ return "", nil, 0, 0, err
+ }
+
+ return disk.FileName, disk.TransferMAC, disk.NumParts, disk.FileSize, nil
+}
+
+// savePart saves the given part to storage keying on its part number.
+func savePart(part []byte, partNum int, kv *versioned.KV) error {
+ obj := &versioned.Object{
+ Version: receivedPartStoreVersion,
+ Timestamp: netTime.Now(),
+ Data: part,
+ }
+
+ return kv.Set(makeReceivedPartKey(partNum), receivedPartStoreVersion, obj)
+}
+
+// loadPart loads the part with the given part number from storage.
+func loadPart(partNum int, kv *versioned.KV) ([]byte, error) {
+ obj, err := kv.Get(makeReceivedPartKey(partNum), receivedPartStoreVersion)
+ if err != nil {
+ return nil, err
+ }
+ return obj.Data, nil
+}
+
+// makeReceivedTransferPrefix generates the unique prefix used on the key value
+// store to store received transfers for the given transfer ID.
+func makeReceivedTransferPrefix(tid *ftCrypto.TransferID) string {
+ return receivedTransferStorePrefix +
+ base64.StdEncoding.EncodeToString(tid.Bytes())
+}
+
+// makeReceivedPartKey generates a storage key for the given part number.
+func makeReceivedPartKey(partNum int) string {
+ return receivedPartStoreKey + strconv.Itoa(partNum)
+}
diff --git a/fileTransfer2/store/receivedTransfer_test.go b/fileTransfer2/store/receivedTransfer_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..2fb0b4b9ab616e4e0f7745e3a5aa41063d0313be
--- /dev/null
+++ b/fileTransfer2/store/receivedTransfer_test.go
@@ -0,0 +1,454 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package store
+
+import (
+ "bytes"
+ "fmt"
+ "gitlab.com/elixxir/client/fileTransfer2/store/cypher"
+ "gitlab.com/elixxir/client/storage/utility"
+ "gitlab.com/elixxir/client/storage/versioned"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/elixxir/ekv"
+ "gitlab.com/xx_network/crypto/csprng"
+ "reflect"
+ "testing"
+)
+
+// Tests that newReceivedTransfer returns a new ReceivedTransfer with the
+// expected values.
+func Test_newReceivedTransfer(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ key, _ := ftCrypto.NewTransferKey(csprng.NewSystemRNG())
+ tid, _ := ftCrypto.NewTransferID(csprng.NewSystemRNG())
+ numFps := uint16(24)
+ parts := generateTestParts(16)
+ fileSize := uint32(len(parts) * len(parts[0]))
+ numParts := uint16(len(parts))
+ rtKv := kv.Prefix(makeReceivedTransferPrefix(&tid))
+
+ cypherManager, err := cypher.NewManager(&key, numFps, rtKv)
+ if err != nil {
+ t.Errorf("Failed to make new cypher manager: %+v", err)
+ }
+ partStatus, err := utility.NewStateVector(
+ rtKv, receivedTransferStatusKey, uint32(numParts))
+ if err != nil {
+ t.Errorf("Failed to make new state vector: %+v", err)
+ }
+
+ expected := &ReceivedTransfer{
+ cypherManager: cypherManager,
+ tid: &tid,
+ fileName: "fileName",
+ transferMAC: []byte("transferMAC"),
+ numParts: numParts,
+ fileSize: fileSize,
+ parts: make([][]byte, numParts),
+ partStatus: partStatus,
+ kv: rtKv,
+ }
+
+ rt, err := newReceivedTransfer(&key, &tid, expected.fileName,
+ expected.transferMAC, numParts, numFps, fileSize, kv)
+ if err != nil {
+ t.Errorf("newReceivedTransfer returned an error: %+v", err)
+ }
+
+ if !reflect.DeepEqual(expected, rt) {
+ t.Errorf("New ReceivedTransfer does not match expected."+
+ "\nexpected: %+v\nreceived: %+v", expected, rt)
+ }
+}
+
+// Tests that ReceivedTransfer.AddPart adds the part to the part list and marks
+// it as received
+func TestReceivedTransfer_AddPart(t *testing.T) {
+ rt, _, _, _ := newTestReceivedTransfer(16, t)
+
+ part := []byte("Part")
+ partNum := 6
+
+ err := rt.AddPart(part, partNum)
+ if err != nil {
+ t.Errorf("Failed to add part: %+v", err)
+ }
+
+ if !bytes.Equal(rt.parts[partNum], part) {
+ t.Errorf("Found incorrect part in list.\nexpected: %q\nreceived: %q",
+ part, rt.parts[partNum])
+ }
+
+ if !rt.partStatus.Used(uint32(partNum)) {
+ t.Errorf("Part #%d not marked as received.", partNum)
+ }
+}
+
+// Tests that ReceivedTransfer.AddPart returns an error if the part number is
+// not within the range of part numbers
+func TestReceivedTransfer_AddPart_PartOutOfRangeError(t *testing.T) {
+ rt, _, _, _ := newTestReceivedTransfer(16, t)
+
+ expectedErr := fmt.Sprintf(errPartOutOfRange, rt.partStatus.GetNumKeys(),
+ rt.partStatus.GetNumKeys()-1)
+
+ err := rt.AddPart([]byte("Part"), int(rt.partStatus.GetNumKeys()))
+ if err == nil || err.Error() != expectedErr {
+ t.Errorf("Failed to get expected error when part number is out of range."+
+ "\nexpected: %s\nreceived: %+v", expectedErr, err)
+ }
+}
+
+// Tests that ReceivedTransfer.GetFile returns the expected file after all the
+// parts are added to the transfer.
+func TestReceivedTransfer_GetFile(t *testing.T) {
+ // Generate parts and make last file part smaller than the rest
+ parts := generateTestParts(16)
+ lastPartLen := 6
+ rt, _, _, _ := newTestReceivedTransfer(uint16(len(parts)), t)
+ rt.fileSize = uint32((len(parts)-1)*len(parts[0]) + lastPartLen)
+
+ for i, p := range parts {
+ err := rt.AddPart(p, i)
+ if err != nil {
+ t.Errorf("Failed to add part #%d: %+v", i, err)
+ }
+ }
+
+ parts[len(parts)-1] = parts[len(parts)-1][:lastPartLen]
+ combinedParts := bytes.Join(parts, nil)
+
+ file := rt.GetFile()
+
+ if !bytes.Equal(file, combinedParts) {
+ t.Errorf("Received file does not match expected."+
+ "\nexpected: %q\nreceived: %q", combinedParts, file)
+ }
+
+}
+
+// Tests that ReceivedTransfer.GetUnusedCyphers returns the correct number of
+// unused cyphers.
+func TestReceivedTransfer_GetUnusedCyphers(t *testing.T) {
+ numParts := uint16(10)
+ rt, _, numFps, _ := newTestReceivedTransfer(numParts, t)
+
+ // Check that all cyphers are returned after initialisation
+ unsentCyphers := rt.GetUnusedCyphers()
+ if len(unsentCyphers) != int(numFps) {
+ t.Errorf("Number of unused cyphers does not match original number of "+
+ "fingerprints when none have been used.\nexpected: %d\nreceived: %d",
+ numFps, len(unsentCyphers))
+ }
+
+ // Use every other part
+ for i := range unsentCyphers {
+ if i%2 == 0 {
+ _, _ = unsentCyphers[i].PopCypher()
+ }
+ }
+
+ // Check that only have the number of parts is returned
+ unsentCyphers = rt.GetUnusedCyphers()
+ if len(unsentCyphers) != int(numFps)/2 {
+ t.Errorf("Number of unused cyphers is not half original number after "+
+ "half have been marked as received.\nexpected: %d\nreceived: %d",
+ numFps/2, len(unsentCyphers))
+ }
+
+ // Use the rest of the parts
+ for i := range unsentCyphers {
+ _, _ = unsentCyphers[i].PopCypher()
+ }
+
+ // Check that no sent parts are returned
+ unsentCyphers = rt.GetUnusedCyphers()
+ if len(unsentCyphers) != 0 {
+ t.Errorf("Number of unused cyphers is not zero after all have been "+
+ "marked as received.\nexpected: %d\nreceived: %d",
+ 0, len(unsentCyphers))
+ }
+}
+
+// Tests that ReceivedTransfer.NumParts returns the correct number of parts.
+func TestReceivedTransfer_NumParts(t *testing.T) {
+ numParts := uint16(16)
+ rt, _, _, _ := newTestReceivedTransfer(numParts, t)
+
+ if rt.NumParts() != numParts {
+ t.Errorf("Incorrect number of parts.\nexpected: %d\nreceived: %d",
+ numParts, rt.NumParts())
+ }
+}
+
+// Tests that ReceivedTransfer.TransferID returns the correct transfer ID.
+func TestReceivedTransfer_TransferID(t *testing.T) {
+ rt, _, _, _ := newTestReceivedTransfer(16, t)
+
+ if rt.TransferID() != rt.tid {
+ t.Errorf("Incorrect transfer ID.\nexpected: %s\nreceived: %s",
+ rt.tid, rt.TransferID())
+ }
+}
+
+// Tests that ReceivedTransfer.FileName returns the correct file name.
+func TestReceivedTransfer_FileName(t *testing.T) {
+ rt, _, _, _ := newTestReceivedTransfer(16, t)
+
+ if rt.FileName() != rt.fileName {
+ t.Errorf("Incorrect transfer ID.\nexpected: %s\nreceived: %s",
+ rt.fileName, rt.FileName())
+ }
+}
+
+// Tests that ReceivedTransfer.NumReceived returns the correct number of
+// received parts.
+func TestReceivedTransfer_NumReceived(t *testing.T) {
+ rt, _, _, _ := newTestReceivedTransfer(16, t)
+
+ if rt.NumReceived() != 0 {
+ t.Errorf("Incorrect number of received parts."+
+ "\nexpected: %d\nreceived: %d", 0, rt.NumReceived())
+ }
+
+ // Add all parts as received
+ for i := 0; i < int(rt.numParts); i++ {
+ _ = rt.AddPart(nil, i)
+ }
+
+ if uint32(rt.NumReceived()) != rt.partStatus.GetNumKeys() {
+ t.Errorf("Incorrect number of received parts."+
+ "\nexpected: %d\nreceived: %d",
+ uint32(rt.NumReceived()), rt.partStatus.GetNumKeys())
+ }
+}
+
+// Tests that the state vector returned by ReceivedTransfer.CopyPartStatusVector
+// has the same values as the original but is a copy.
+func TestReceivedTransfer_CopyPartStatusVector(t *testing.T) {
+ rt, _, _, _ := newTestReceivedTransfer(64, t)
+
+ // Check that the vectors have the same unused parts
+ partStatus := rt.CopyPartStatusVector()
+ if !reflect.DeepEqual(
+ partStatus.GetUnusedKeyNums(), rt.partStatus.GetUnusedKeyNums()) {
+ t.Errorf("Copied part status does not match original."+
+ "\nexpected: %v\nreceived: %v",
+ rt.partStatus.GetUnusedKeyNums(), partStatus.GetUnusedKeyNums())
+ }
+
+ // Modify the state
+ _ = rt.AddPart([]byte("hello"), 5)
+
+ // Check that the copied state is different
+ if reflect.DeepEqual(
+ partStatus.GetUnusedKeyNums(), rt.partStatus.GetUnusedKeyNums()) {
+ t.Errorf("Old copied part status matches new status."+
+ "\nexpected: %v\nreceived: %v",
+ rt.partStatus.GetUnusedKeyNums(), partStatus.GetUnusedKeyNums())
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Storage Functions //
+////////////////////////////////////////////////////////////////////////////////
+
+// Tests that a ReceivedTransfer loaded via loadReceivedTransfer matches the
+// original.
+func Test_loadReceivedTransfer(t *testing.T) {
+ parts := generateTestParts(16)
+ rt, _, _, kv := newTestReceivedTransfer(uint16(len(parts)), t)
+
+ for i, p := range parts {
+ if i%2 == 0 {
+
+ err := rt.AddPart(p, i)
+ if err != nil {
+ t.Errorf("Failed to add part #%d: %+v", i, err)
+ }
+ }
+ }
+
+ loadedRt, err := loadReceivedTransfer(rt.tid, kv)
+ if err != nil {
+ t.Errorf("Failed to load ReceivedTransfer: %+v", err)
+ }
+
+ if !reflect.DeepEqual(rt, loadedRt) {
+ t.Errorf("Loaded ReceivedTransfer does not match original."+
+ "\nexpected: %+v\nreceived: %+v", rt, loadedRt)
+ }
+}
+
+// Tests that ReceivedTransfer.Delete deletes the storage backend of the
+// ReceivedTransfer and that it cannot be loaded again.
+func TestReceivedTransfer_Delete(t *testing.T) {
+ rt, _, _, kv := newTestReceivedTransfer(64, t)
+
+ err := rt.Delete()
+ if err != nil {
+ t.Errorf("Delete returned an error: %+v", err)
+ }
+
+ _, err = loadSentTransfer(rt.tid, kv)
+ if err == nil {
+ t.Errorf("Loaded received transfer that was deleted.")
+ }
+}
+
+// Tests that the fields saved by ReceivedTransfer.save can be loaded from
+// storage.
+func TestReceivedTransfer_save(t *testing.T) {
+ rt, _, _, _ := newTestReceivedTransfer(64, t)
+
+ err := rt.save()
+ if err != nil {
+ t.Errorf("save returned an error: %+v", err)
+ }
+
+ _, err = rt.kv.Get(receivedTransferStoreKey, receivedTransferStoreVersion)
+ if err != nil {
+ t.Errorf("Failed to load saved ReceivedTransfer: %+v", err)
+ }
+}
+
+// newTestReceivedTransfer creates a new ReceivedTransfer for testing.
+func newTestReceivedTransfer(numParts uint16, t *testing.T) (
+ *ReceivedTransfer, *ftCrypto.TransferKey, uint16, *versioned.KV) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ key, _ := ftCrypto.NewTransferKey(csprng.NewSystemRNG())
+ tid, _ := ftCrypto.NewTransferID(csprng.NewSystemRNG())
+ transferMAC := []byte("I am a transfer MAC")
+ numFps := 2 * numParts
+ fileName := "helloFile"
+ parts := generateTestParts(numParts)
+ fileSize := uint32(len(parts) * len(parts[0]))
+
+ st, err := newReceivedTransfer(
+ &key, &tid, fileName, transferMAC, numParts, numFps, fileSize, kv)
+ if err != nil {
+ t.Errorf("Failed to make new SentTransfer: %+v", err)
+ }
+
+ return st, &key, numFps, kv
+}
+
+// Tests that a ReceivedTransfer marshalled via ReceivedTransfer.marshal and
+// unmarshalled via unmarshalReceivedTransfer matches the original.
+func TestReceivedTransfer_marshal_unmarshalReceivedTransfer(t *testing.T) {
+ rt := &ReceivedTransfer{
+ fileName: "transferName",
+ transferMAC: []byte("I am a transfer MAC"),
+ numParts: 153,
+ fileSize: 735,
+ }
+
+ data, err := rt.marshal()
+ if err != nil {
+ t.Errorf("marshal returned an error: %+v", err)
+ }
+
+ fileName, transferMac, numParts, fileSize, err :=
+ unmarshalReceivedTransfer(data)
+ if err != nil {
+ t.Errorf("Failed to unmarshal SentTransfer: %+v", err)
+ }
+
+ if rt.fileName != fileName {
+ t.Errorf("Incorrect file name.\nexpected: %q\nreceived: %q",
+ rt.fileName, fileName)
+ }
+
+ if !bytes.Equal(rt.transferMAC, transferMac) {
+ t.Errorf("Incorrect transfer MAC.\nexpected: %s\nreceived: %s",
+ rt.transferMAC, transferMac)
+ }
+
+ if rt.numParts != numParts {
+ t.Errorf("Incorrect number of parts.\nexpected: %d\nreceived: %d",
+ rt.numParts, numParts)
+ }
+
+ if rt.fileSize != fileSize {
+ t.Errorf("Incorrect file size.\nexpected: %d\nreceived: %d",
+ rt.fileSize, fileSize)
+ }
+}
+
+// Tests that the part saved to storage via savePart can be loaded.
+func Test_savePart_loadPart(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ part := []byte("I am a part.")
+ partNum := 18
+
+ err := savePart(part, partNum, kv)
+ if err != nil {
+ t.Errorf("Failed to save part: %+v", err)
+ }
+
+ loadedPart, err := loadPart(partNum, kv)
+ if err != nil {
+ t.Errorf("Failed to load part: %+v", err)
+ }
+
+ if !bytes.Equal(part, loadedPart) {
+ t.Errorf("Loaded part does not match original."+
+ "\nexpected: %q\nreceived: %q", part, loadedPart)
+ }
+}
+
+// Consistency test of makeReceivedTransferPrefix.
+func Test_makeReceivedTransferPrefix_Consistency(t *testing.T) {
+ expectedPrefixes := []string{
+ "ReceivedFileTransferStore/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "ReceivedFileTransferStore/AQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "ReceivedFileTransferStore/AgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "ReceivedFileTransferStore/AwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "ReceivedFileTransferStore/BAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "ReceivedFileTransferStore/BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "ReceivedFileTransferStore/BgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "ReceivedFileTransferStore/BwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "ReceivedFileTransferStore/CAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "ReceivedFileTransferStore/CQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ }
+
+ for i, expected := range expectedPrefixes {
+ tid := ftCrypto.TransferID{byte(i)}
+ prefix := makeReceivedTransferPrefix(&tid)
+
+ if expected != prefix {
+ t.Errorf("Prefix #%d does not match expected."+
+ "\nexpected: %q\nreceived: %q", i, expected, prefix)
+ }
+ }
+}
+
+// Consistency test of makeReceivedPartKey.
+func Test_makeReceivedPartKey_Consistency(t *testing.T) {
+ expectedKeys := []string{
+ "receivedPart#0", "receivedPart#1", "receivedPart#2", "receivedPart#3",
+ "receivedPart#4", "receivedPart#5", "receivedPart#6", "receivedPart#7",
+ "receivedPart#8", "receivedPart#9",
+ }
+
+ for i, expected := range expectedKeys {
+ key := makeReceivedPartKey(i)
+
+ if expected != key {
+ t.Errorf("Key #%d does not match expected."+
+ "\nexpected: %q\nreceived: %q", i, expected, key)
+ }
+ }
+}
diff --git a/fileTransfer2/store/received_test.go b/fileTransfer2/store/received_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..6ce990a2523b87c9726e082f0314d4f4aad77823
--- /dev/null
+++ b/fileTransfer2/store/received_test.go
@@ -0,0 +1,262 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package store
+
+import (
+ "bytes"
+ "fmt"
+ "gitlab.com/elixxir/client/storage/versioned"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/elixxir/ekv"
+ "gitlab.com/xx_network/crypto/csprng"
+ "reflect"
+ "sort"
+ "strconv"
+ "testing"
+)
+
+// Tests that NewOrLoadReceived returns a new Received when none exist in
+// storage and that the list of incomplete transfers is nil.
+func TestNewOrLoadReceived_New(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ expected := &Received{
+ transfers: make(map[ftCrypto.TransferID]*ReceivedTransfer),
+ kv: kv.Prefix(receivedTransfersStorePrefix),
+ }
+
+ r, incompleteTransfers, err := NewOrLoadReceived(kv)
+ if err != nil {
+ t.Errorf("NewOrLoadReceived returned an error: %+v", err)
+ }
+
+ if !reflect.DeepEqual(expected, r) {
+ t.Errorf("New Received does not match expected."+
+ "\nexpected: %+v\nreceived: %+v", expected, r)
+ }
+
+ if incompleteTransfers != nil {
+ t.Errorf("List of incomplete transfers should be nil when not "+
+ "loading: %+v", incompleteTransfers)
+ }
+}
+
+// Tests that NewOrLoadReceived returns a loaded Received when one exist in
+// storage and that the list of incomplete transfers is correct.
+func TestNewOrLoadReceived_Load(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ r, _, err := NewOrLoadReceived(kv)
+ if err != nil {
+ t.Errorf("Failed to make new Received: %+v", err)
+ }
+ var expectedIncompleteTransfers []*ReceivedTransfer
+
+ // Create and add transfers to map and save
+ for i := 0; i < 2; i++ {
+ key, _ := ftCrypto.NewTransferKey(csprng.NewSystemRNG())
+ tid, _ := ftCrypto.NewTransferID(csprng.NewSystemRNG())
+ rt, err2 := r.AddTransfer(&key, &tid, "file"+strconv.Itoa(i),
+ []byte("transferMAC"+strconv.Itoa(i)), 10, 20, 128)
+ if err2 != nil {
+ t.Errorf("Failed to add transfer #%d: %+v", i, err2)
+ }
+ expectedIncompleteTransfers = append(expectedIncompleteTransfers, rt)
+ }
+ if err = r.save(); err != nil {
+ t.Errorf("Failed to make save filled Receivced: %+v", err)
+ }
+
+ // Load Received
+ loadedReceived, incompleteTransfers, err := NewOrLoadReceived(kv)
+ if err != nil {
+ t.Errorf("Failed to load Received: %+v", err)
+ }
+
+ // Check that the loaded Received matches original
+ if !reflect.DeepEqual(r, loadedReceived) {
+ t.Errorf("Loaded Received does not match original."+
+ "\nexpected: %#v\nreceived: %#v", r, loadedReceived)
+ }
+
+ sort.Slice(incompleteTransfers, func(i, j int) bool {
+ return bytes.Compare(incompleteTransfers[i].TransferID()[:],
+ incompleteTransfers[j].TransferID()[:]) == -1
+ })
+
+ sort.Slice(expectedIncompleteTransfers, func(i, j int) bool {
+ return bytes.Compare(expectedIncompleteTransfers[i].TransferID()[:],
+ expectedIncompleteTransfers[j].TransferID()[:]) == -1
+ })
+
+ // Check that the incomplete transfers matches expected
+ if !reflect.DeepEqual(expectedIncompleteTransfers, incompleteTransfers) {
+ t.Errorf("Incorrect incomplete transfers.\nexpected: %v\nreceived: %v",
+ expectedIncompleteTransfers, incompleteTransfers)
+ }
+}
+
+// Tests that Received.AddTransfer makes a new transfer and adds it to the list.
+func TestReceived_AddTransfer(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ r, _, _ := NewOrLoadReceived(kv)
+
+ key, _ := ftCrypto.NewTransferKey(csprng.NewSystemRNG())
+ tid, _ := ftCrypto.NewTransferID(csprng.NewSystemRNG())
+
+ rt, err := r.AddTransfer(
+ &key, &tid, "file", []byte("transferMAC"), 10, 20, 128)
+ if err != nil {
+ t.Errorf("Failed to add new transfer: %+v", err)
+ }
+
+ // Check that the transfer was added
+ if _, exists := r.transfers[*rt.tid]; !exists {
+ t.Errorf("No transfer with ID %s exists.", rt.tid)
+ }
+}
+
+// Tests that Received.AddTransfer returns an error when adding a transfer ID
+// that already exists.
+func TestReceived_AddTransfer_TransferAlreadyExists(t *testing.T) {
+ tid := ftCrypto.TransferID{0}
+ r := &Received{
+ transfers: map[ftCrypto.TransferID]*ReceivedTransfer{tid: nil},
+ }
+
+ expectedErr := fmt.Sprintf(errAddExistingReceivedTransfer, tid)
+ _, err := r.AddTransfer(nil, &tid, "", nil, 0, 0, 0)
+ if err == nil || err.Error() != expectedErr {
+ t.Errorf("Received unexpected error when adding transfer that already "+
+ "exists.\nexpected: %s\nreceived: %+v", expectedErr, err)
+ }
+}
+
+// Tests that Received.GetTransfer returns the expected transfer.
+func TestReceived_GetTransfer(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ r, _, _ := NewOrLoadReceived(kv)
+
+ key, _ := ftCrypto.NewTransferKey(csprng.NewSystemRNG())
+ tid, _ := ftCrypto.NewTransferID(csprng.NewSystemRNG())
+
+ rt, err := r.AddTransfer(
+ &key, &tid, "file", []byte("transferMAC"), 10, 20, 128)
+ if err != nil {
+ t.Errorf("Failed to add new transfer: %+v", err)
+ }
+
+ // Check that the transfer was added
+ receivedRt, exists := r.GetTransfer(rt.tid)
+ if !exists {
+ t.Errorf("No transfer with ID %s exists.", rt.tid)
+ }
+
+ if !reflect.DeepEqual(rt, receivedRt) {
+ t.Errorf("Received ReceivedTransfer does not match expected."+
+ "\nexpected: %+v\nreceived: %+v", rt, receivedRt)
+ }
+}
+
+// Tests that Sent.RemoveTransfer removes the transfer from the list.
+func TestReceived_RemoveTransfer(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ r, _, _ := NewOrLoadReceived(kv)
+
+ key, _ := ftCrypto.NewTransferKey(csprng.NewSystemRNG())
+ tid, _ := ftCrypto.NewTransferID(csprng.NewSystemRNG())
+
+ rt, err := r.AddTransfer(
+ &key, &tid, "file", []byte("transferMAC"), 10, 20, 128)
+ if err != nil {
+ t.Errorf("Failed to add new transfer: %+v", err)
+ }
+
+ // Delete the transfer
+ err = r.RemoveTransfer(rt.tid)
+ if err != nil {
+ t.Errorf("RemoveTransfer returned an error: %+v", err)
+ }
+
+ // Check that the transfer was deleted
+ _, exists := r.GetTransfer(rt.tid)
+ if exists {
+ t.Errorf("Transfer %s exists.", rt.tid)
+ }
+
+ // Remove transfer that was already removed
+ err = r.RemoveTransfer(rt.tid)
+ if err != nil {
+ t.Errorf("RemoveTransfer returned an error: %+v", err)
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Storage Functions //
+////////////////////////////////////////////////////////////////////////////////
+
+// Tests that Received.save saves the transfer ID list to storage by trying to
+// load it after a save.
+func TestReceived_save(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ r, _, _ := NewOrLoadReceived(kv)
+ r.transfers = map[ftCrypto.TransferID]*ReceivedTransfer{
+ ftCrypto.TransferID{0}: nil, ftCrypto.TransferID{1}: nil,
+ ftCrypto.TransferID{2}: nil, ftCrypto.TransferID{3}: nil,
+ }
+
+ err := r.save()
+ if err != nil {
+ t.Errorf("Failed to save transfer ID list: %+v", err)
+ }
+
+ _, err = r.kv.Get(receivedTransfersStoreKey, receivedTransfersStoreVersion)
+ if err != nil {
+ t.Errorf("Failed to load transfer ID list: %+v", err)
+ }
+}
+
+// Tests that the transfer IDs keys in the map marshalled by
+// marshalReceivedTransfersMap and unmarshalled by unmarshalTransferIdList match
+// the original.
+func Test_marshalReceivedTransfersMap_unmarshalTransferIdList(t *testing.T) {
+ // Build map of transfer IDs
+ transfers := make(map[ftCrypto.TransferID]*ReceivedTransfer, 10)
+ for i := 0; i < 10; i++ {
+ tid, _ := ftCrypto.NewTransferID(csprng.NewSystemRNG())
+ transfers[tid] = nil
+ }
+
+ data, err := marshalReceivedTransfersMap(transfers)
+ if err != nil {
+ t.Errorf("marshalReceivedTransfersMap returned an error: %+v", err)
+ }
+
+ tidList, err := unmarshalTransferIdList(data)
+ if err != nil {
+ t.Errorf("unmarshalSentTransfer returned an error: %+v", err)
+ }
+
+ for _, tid := range tidList {
+ if _, exists := transfers[tid]; exists {
+ delete(transfers, tid)
+ } else {
+ t.Errorf("Transfer %s does not exist in list.", tid)
+ }
+ }
+
+ if len(transfers) != 0 {
+ t.Errorf("%d transfers not in unmarshalled list: %v",
+ len(transfers), transfers)
+ }
+}
diff --git a/fileTransfer2/store/sent.go b/fileTransfer2/store/sent.go
new file mode 100644
index 0000000000000000000000000000000000000000..9ca7f482e53540adff504c932ad39219225e39fa
--- /dev/null
+++ b/fileTransfer2/store/sent.go
@@ -0,0 +1,192 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package store
+
+import (
+ "encoding/json"
+ "github.com/pkg/errors"
+ jww "github.com/spf13/jwalterweatherman"
+ "gitlab.com/elixxir/client/storage/versioned"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/elixxir/ekv"
+ "gitlab.com/xx_network/primitives/id"
+ "gitlab.com/xx_network/primitives/netTime"
+ "sync"
+)
+
+// Storage keys and versions.
+const (
+ sentTransfersStorePrefix = "SentFileTransfersPrefix"
+ sentTransfersStoreKey = "SentFileTransfers"
+ sentTransfersStoreVersion = 0
+)
+
+// Error messages.
+const (
+ // NewOrLoadSent
+ errLoadSent = "error loading sent transfer list from storage: %+v"
+ errUnmarshalSent = "could not unmarshal sent transfer list: %+v"
+ warnLoadSentTransfer = "[FT] Failed to load sent transfer %d of %d with ID %s: %+v"
+ errLoadAllSentTransfer = "failed to load all %d transfers"
+
+ // Sent.AddTransfer
+ errAddExistingSentTransfer = "sent transfer with ID %s already exists in map."
+ errNewSentTransfer = "failed to make new sent transfer: %+v"
+)
+
+// Sent contains a list of all sent transfers.
+type Sent struct {
+ transfers map[ftCrypto.TransferID]*SentTransfer
+
+ mux sync.RWMutex
+ kv *versioned.KV
+}
+
+// NewOrLoadSent attempts to load Sent from storage. Or if none exist, then a
+// new Sent is returned. If running transfers were loaded from storage, a list
+// of unsent parts is returned.
+func NewOrLoadSent(kv *versioned.KV) (*Sent, []Part, error) {
+ s := &Sent{
+ transfers: make(map[ftCrypto.TransferID]*SentTransfer),
+ kv: kv.Prefix(sentTransfersStorePrefix),
+ }
+
+ obj, err := s.kv.Get(sentTransfersStoreKey, sentTransfersStoreVersion)
+ if err != nil {
+ if !ekv.Exists(err) {
+ // Return the new Sent if none exists in storage
+ return s, nil, nil
+ } else {
+ // Return other errors
+ return nil, nil, errors.Errorf(errLoadSent, err)
+ }
+ }
+
+ // Load list of saved sent transfers from storage
+ tidList, err := unmarshalTransferIdList(obj.Data)
+ if err != nil {
+ return nil, nil, errors.Errorf(errUnmarshalSent, err)
+ }
+
+ // Load sent transfers from storage
+ var errCount int
+ var unsentParts []Part
+ for i := range tidList {
+ tid := tidList[i]
+ s.transfers[tid], err = loadSentTransfer(&tid, s.kv)
+ if err != nil {
+ jww.WARN.Printf(warnLoadSentTransfer, i, len(tidList), tid, err)
+ errCount++
+ continue
+ }
+
+ if s.transfers[tid].Status() == Running {
+ unsentParts =
+ append(unsentParts, s.transfers[tid].GetUnsentParts()...)
+ }
+ }
+
+ // Return an error if all transfers failed to load
+ if errCount == len(tidList) {
+ return nil, nil, errors.Errorf(errLoadAllSentTransfer, len(tidList))
+ }
+
+ return s, unsentParts, nil
+}
+
+// AddTransfer creates a SentTransfer and adds it to the map keyed on its
+// transfer ID.
+func (s *Sent) AddTransfer(recipient *id.ID, key *ftCrypto.TransferKey,
+ tid *ftCrypto.TransferID, fileName string, parts [][]byte, numFps uint16) (
+ *SentTransfer, error) {
+ s.mux.Lock()
+ defer s.mux.Unlock()
+
+ _, exists := s.transfers[*tid]
+ if exists {
+ return nil, errors.Errorf(errAddExistingSentTransfer, tid)
+ }
+
+ st, err := newSentTransfer(recipient, key, tid, fileName, parts, numFps, s.kv)
+ if err != nil {
+ return nil, errors.Errorf(errNewSentTransfer, tid)
+ }
+
+ s.transfers[*tid] = st
+
+ return st, s.save()
+}
+
+// GetTransfer returns the SentTransfer with the desiccated transfer ID or false
+// if none exists.
+func (s *Sent) GetTransfer(tid *ftCrypto.TransferID) (*SentTransfer, bool) {
+ s.mux.RLock()
+ defer s.mux.RUnlock()
+
+ st, exists := s.transfers[*tid]
+ return st, exists
+}
+
+// RemoveTransfer removes the transfer from the map. If no transfer exists,
+// returns nil. Only errors due to saving to storage are returned.
+func (s *Sent) RemoveTransfer(tid *ftCrypto.TransferID) error {
+ s.mux.Lock()
+ defer s.mux.Unlock()
+
+ _, exists := s.transfers[*tid]
+ if !exists {
+ return nil
+ }
+
+ delete(s.transfers, *tid)
+ return s.save()
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Storage Functions //
+////////////////////////////////////////////////////////////////////////////////
+
+// save stores a list of transfer IDs in the map to storage.
+func (s *Sent) save() error {
+ data, err := marshalSentTransfersMap(s.transfers)
+ if err != nil {
+ return err
+ }
+
+ obj := &versioned.Object{
+ Version: sentTransfersStoreVersion,
+ Timestamp: netTime.Now(),
+ Data: data,
+ }
+
+ return s.kv.Set(sentTransfersStoreKey, sentTransfersStoreVersion, obj)
+}
+
+// marshalSentTransfersMap serialises the list of transfer IDs from a
+// SentTransfer map.
+func marshalSentTransfersMap(transfers map[ftCrypto.TransferID]*SentTransfer) (
+ []byte, error) {
+ tidList := make([]ftCrypto.TransferID, 0, len(transfers))
+
+ for tid := range transfers {
+ tidList = append(tidList, tid)
+ }
+
+ return json.Marshal(tidList)
+}
+
+// unmarshalTransferIdList deserializes the data into a list of transfer IDs.
+func unmarshalTransferIdList(data []byte) ([]ftCrypto.TransferID, error) {
+ var tidList []ftCrypto.TransferID
+ err := json.Unmarshal(data, &tidList)
+ if err != nil {
+ return nil, err
+ }
+
+ return tidList, nil
+}
diff --git a/fileTransfer2/store/sentTransfer.go b/fileTransfer2/store/sentTransfer.go
new file mode 100644
index 0000000000000000000000000000000000000000..c421c86d4d7c68fca4128777fdfc4f677d022756
--- /dev/null
+++ b/fileTransfer2/store/sentTransfer.go
@@ -0,0 +1,346 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package store
+
+import (
+ "encoding/base64"
+ "encoding/json"
+ "github.com/pkg/errors"
+ jww "github.com/spf13/jwalterweatherman"
+ "gitlab.com/elixxir/client/fileTransfer2/store/cypher"
+ "gitlab.com/elixxir/client/storage/utility"
+ "gitlab.com/elixxir/client/storage/versioned"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/xx_network/primitives/id"
+ "gitlab.com/xx_network/primitives/netTime"
+ "sync"
+)
+
+// Storage keys and versions.
+const (
+ sentTransferStorePrefix = "SentFileTransferStore/"
+ sentTransferStoreKey = "SentTransfer"
+ sentTransferStoreVersion = 0
+ sentTransferStatusKey = "SentPartStatusVector"
+)
+
+// Error messages.
+const (
+ // newSentTransfer
+ errStNewCypherManager = "failed to create new cypher manager: %+v"
+ errStNewPartStatusVector = "failed to create new state vector for part statuses: %+v"
+
+ // SentTransfer.getPartData
+ errNoPartNum = "no part with part number %d exists in transfer %s (%q)"
+
+ // loadSentTransfer
+ errStLoadCypherManager = "failed to load cypher manager from storage: %+v"
+ errStLoadFields = "failed to load recipient, status, and parts list: %+v"
+ errStUnmarshalFields = "failed to unmarshal recipient, status, and parts list: %+v"
+ errStLoadPartStatusVector = "failed to load state vector for part statuses: %+v"
+
+ // SentTransfer.Delete
+ errStDeleteCypherManager = "failed to delete cypherManager: %+v"
+ errStDeleteSentTransfer = "failed to delete recipient ID, status, and file parts: %+v"
+ errStDeletePartStatus = "failed to delete part status multi state vector: %+v"
+
+ // SentTransfer.save
+ errMarshalSentTransfer = "failed to marshal: %+v"
+)
+
+// SentTransfer contains information and progress data for sending or sent file
+// transfer.
+type SentTransfer struct {
+ // Tracks cyphers for each part
+ cypherManager *cypher.Manager
+
+ // The ID of the transfer
+ tid *ftCrypto.TransferID
+
+ // User given name to file
+ fileName string
+
+ // ID of the recipient of the file transfer
+ recipient *id.ID
+
+ // The number of file parts in the file
+ numParts uint16
+
+ // Indicates the status of the transfer
+ status TransferStatus
+
+ // List of all file parts in order to send
+ parts [][]byte
+
+ // Stores the status of each part in a bitstream format
+ partStatus *utility.StateVector
+
+ mux sync.RWMutex
+ kv *versioned.KV
+}
+
+// newSentTransfer generates a new SentTransfer with the specified transfer key,
+// transfer ID, and parts.
+func newSentTransfer(recipient *id.ID, key *ftCrypto.TransferKey,
+ tid *ftCrypto.TransferID, fileName string, parts [][]byte, numFps uint16,
+ kv *versioned.KV) (*SentTransfer, error) {
+ kv = kv.Prefix(makeSentTransferPrefix(tid))
+
+ // Create new cypher manager
+ cypherManager, err := cypher.NewManager(key, numFps, kv)
+ if err != nil {
+ return nil, errors.Errorf(errStNewCypherManager, err)
+ }
+
+ // Create new state vector for storing statuses of arrived parts
+ partStatus, err := utility.NewStateVector(
+ kv, sentTransferStatusKey, uint32(len(parts)))
+ if err != nil {
+ return nil, errors.Errorf(errStNewPartStatusVector, err)
+ }
+
+ st := &SentTransfer{
+ cypherManager: cypherManager,
+ tid: tid,
+ fileName: fileName,
+ recipient: recipient,
+ numParts: uint16(len(parts)),
+ status: Running,
+ parts: parts,
+ partStatus: partStatus,
+ kv: kv,
+ }
+
+ return st, st.save()
+}
+
+// GetUnsentParts builds a list of all unsent parts, each in a Part object.
+func (st *SentTransfer) GetUnsentParts() []Part {
+ unusedPartNumbers := st.partStatus.GetUnusedKeyNums()
+ partList := make([]Part, len(unusedPartNumbers))
+
+ for i, partNum := range unusedPartNumbers {
+ partList[i] = Part{
+ transfer: st,
+ cypherManager: st.cypherManager,
+ partNum: uint16(partNum),
+ }
+ }
+
+ return partList
+}
+
+// getPartData returns the part data from the given part number.
+func (st *SentTransfer) getPartData(partNum uint16) []byte {
+ if int(partNum) > len(st.parts)-1 {
+ jww.FATAL.Panicf(errNoPartNum, partNum, st.tid, st.fileName)
+ }
+
+ return st.parts[partNum]
+}
+
+// markArrived marks the status of the given part numbers as arrived. When the
+// last part is marked arrived, the transfer is marked as completed.
+func (st *SentTransfer) markArrived(partNum uint16) {
+ st.mux.Lock()
+ defer st.mux.Unlock()
+
+ st.partStatus.Use(uint32(partNum))
+
+ // Mark transfer completed if all parts arrived
+ if st.partStatus.GetNumUsed() == uint32(st.numParts) {
+ st.status = Completed
+ }
+}
+
+// markTransferFailed sets the transfer as failed. Only call this if no more
+// retries are available.
+func (st *SentTransfer) markTransferFailed() {
+ st.mux.Lock()
+ defer st.mux.Unlock()
+ st.status = Failed
+}
+
+// Status returns the status of the transfer.
+func (st *SentTransfer) Status() TransferStatus {
+ st.mux.RLock()
+ defer st.mux.RUnlock()
+ return st.status
+}
+
+// NumParts returns the total number of file parts in the transfer.
+func (st *SentTransfer) NumParts() uint16 {
+ return st.numParts
+}
+
+// TransferID returns the transfer's ID.
+func (st *SentTransfer) TransferID() *ftCrypto.TransferID {
+ return st.tid
+}
+
+// FileName returns the transfer's file name.
+func (st *SentTransfer) FileName() string {
+ return st.fileName
+}
+
+// Recipient returns the transfer's recipient ID.
+func (st *SentTransfer) Recipient() *id.ID {
+ return st.recipient
+}
+
+// NumArrived returns the number of parts that have arrived.
+func (st *SentTransfer) NumArrived() uint16 {
+ return uint16(st.partStatus.GetNumUsed())
+}
+
+// CopyPartStatusVector returns a copy of the part status vector that can be
+// used to look up the current status of parts. Note that the statuses are from
+// when this function is called and not realtime.
+func (st *SentTransfer) CopyPartStatusVector() *utility.StateVector {
+ return st.partStatus.DeepCopy()
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Storage Functions //
+////////////////////////////////////////////////////////////////////////////////
+
+// loadSentTransfer loads the SentTransfer with the given transfer ID from
+// storage.
+func loadSentTransfer(tid *ftCrypto.TransferID, kv *versioned.KV) (
+ *SentTransfer, error) {
+ kv = kv.Prefix(makeSentTransferPrefix(tid))
+
+ // Load cypher manager
+ cypherManager, err := cypher.LoadManager(kv)
+ if err != nil {
+ return nil, errors.Errorf(errStLoadCypherManager, err)
+ }
+
+ // Load fileName, recipient ID, status, and file parts
+ obj, err := kv.Get(sentTransferStoreKey, sentTransferStoreVersion)
+ if err != nil {
+ return nil, errors.Errorf(errStLoadFields, err)
+ }
+
+ fileName, recipient, status, parts, err := unmarshalSentTransfer(obj.Data)
+ if err != nil {
+ return nil, errors.Errorf(errStUnmarshalFields, err)
+ }
+
+ // Load state vector for storing statuses of arrived parts
+ partStatus, err := utility.LoadStateVector(kv, sentTransferStatusKey)
+ if err != nil {
+ return nil, errors.Errorf(errStLoadPartStatusVector, err)
+ }
+
+ st := &SentTransfer{
+ cypherManager: cypherManager,
+ tid: tid,
+ fileName: fileName,
+ recipient: recipient,
+ numParts: uint16(len(parts)),
+ status: status,
+ parts: parts,
+ partStatus: partStatus,
+ kv: kv,
+ }
+
+ return st, nil
+}
+
+// Delete deletes all data in the SentTransfer from storage.
+func (st *SentTransfer) Delete() error {
+ st.mux.Lock()
+ defer st.mux.Unlock()
+
+ // Delete cypher manager
+ err := st.cypherManager.Delete()
+ if err != nil {
+ return errors.Errorf(errStDeleteCypherManager, err)
+ }
+
+ // Delete recipient ID, status, and file parts
+ err = st.kv.Delete(sentTransferStoreKey, sentTransferStoreVersion)
+ if err != nil {
+ return errors.Errorf(errStDeleteSentTransfer, err)
+ }
+
+ // Delete part status multi state vector
+ err = st.partStatus.Delete()
+ if err != nil {
+ return errors.Errorf(errStDeletePartStatus, err)
+ }
+
+ return nil
+}
+
+// save stores all fields in SentTransfer that do not have their own storage
+// (recipient ID, status, and file parts) to storage.
+func (st *SentTransfer) save() error {
+ data, err := st.marshal()
+ if err != nil {
+ return errors.Errorf(errMarshalSentTransfer, err)
+ }
+
+ obj := &versioned.Object{
+ Version: sentTransferStoreVersion,
+ Timestamp: netTime.Now(),
+ Data: data,
+ }
+
+ return st.kv.Set(sentTransferStoreKey, sentTransferStoreVersion, obj)
+}
+
+// sentTransferDisk structure is used to marshal and unmarshal SentTransfer
+// fields to/from storage.
+type sentTransferDisk struct {
+ FileName string
+ Recipient *id.ID
+ Status TransferStatus
+ Parts [][]byte
+}
+
+// marshal serialises the SentTransfer's fileName, recipient, status, and parts
+// list.
+func (st *SentTransfer) marshal() ([]byte, error) {
+ disk := sentTransferDisk{
+ FileName: st.fileName,
+ Recipient: st.recipient,
+ Status: st.status,
+ Parts: st.parts,
+ }
+
+ return json.Marshal(disk)
+}
+
+// unmarshalSentTransfer deserializes the data into a fileName, recipient,
+// status, and parts list.
+func unmarshalSentTransfer(data []byte) (fileName string, recipient *id.ID,
+ status TransferStatus, parts [][]byte, err error) {
+ var disk sentTransferDisk
+ err = json.Unmarshal(data, &disk)
+ if err != nil {
+ return "", nil, 0, nil, err
+ }
+
+ return disk.FileName, disk.Recipient, disk.Status, disk.Parts, nil
+}
+
+// makeSentTransferPrefix generates the unique prefix used on the key value
+// store to store sent transfers for the given transfer ID.
+func makeSentTransferPrefix(tid *ftCrypto.TransferID) string {
+ return sentTransferStorePrefix +
+ base64.StdEncoding.EncodeToString(tid.Bytes())
+}
diff --git a/fileTransfer2/store/sentTransfer_test.go b/fileTransfer2/store/sentTransfer_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..254ba213a122bbf8b047494432dca0e13380be6e
--- /dev/null
+++ b/fileTransfer2/store/sentTransfer_test.go
@@ -0,0 +1,480 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package store
+
+import (
+ "bytes"
+ "fmt"
+ "gitlab.com/elixxir/client/fileTransfer2/store/cypher"
+ "gitlab.com/elixxir/client/fileTransfer2/store/fileMessage"
+ "gitlab.com/elixxir/client/storage/utility"
+ "gitlab.com/elixxir/client/storage/versioned"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/elixxir/ekv"
+ "gitlab.com/elixxir/primitives/format"
+ "gitlab.com/xx_network/crypto/csprng"
+ "gitlab.com/xx_network/primitives/id"
+ "reflect"
+ "strconv"
+ "testing"
+)
+
+// Tests that newSentTransfer returns a new SentTransfer with the expected
+// values.
+func Test_newSentTransfer(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ key, _ := ftCrypto.NewTransferKey(csprng.NewSystemRNG())
+ tid, _ := ftCrypto.NewTransferID(csprng.NewSystemRNG())
+ numFps := uint16(24)
+ parts := [][]byte{[]byte("hello"), []byte("hello"), []byte("hello")}
+ stKv := kv.Prefix(makeSentTransferPrefix(&tid))
+
+ cypherManager, err := cypher.NewManager(&key, numFps, stKv)
+ if err != nil {
+ t.Errorf("Failed to make new cypher manager: %+v", err)
+ }
+ partStatus, err := utility.NewStateVector(
+ stKv, sentTransferStatusKey, uint32(len(parts)))
+ if err != nil {
+ t.Errorf("Failed to make new state vector: %+v", err)
+ }
+
+ expected := &SentTransfer{
+ cypherManager: cypherManager,
+ tid: &tid,
+ fileName: "file",
+ recipient: id.NewIdFromString("user", id.User, t),
+ numParts: uint16(len(parts)),
+ status: Running,
+ parts: parts,
+ partStatus: partStatus,
+ kv: stKv,
+ }
+
+ st, err := newSentTransfer(
+ expected.recipient, &key, &tid, expected.fileName, parts, numFps, kv)
+ if err != nil {
+ t.Errorf("newSentTransfer returned an error: %+v", err)
+ }
+
+ if !reflect.DeepEqual(expected, st) {
+ t.Errorf("New SentTransfer does not match expected."+
+ "\nexpected: %+v\nreceived: %+v", expected, st)
+ }
+}
+
+// Tests that SentTransfer.GetUnsentParts returns the correct number of unsent
+// parts
+func TestSentTransfer_GetUnsentParts(t *testing.T) {
+ numParts := uint16(10)
+ st, _, _, _, _ := newTestSentTransfer(numParts, t)
+
+ // Check that all parts are returned after initialisation
+ unsentParts := st.GetUnsentParts()
+ if len(unsentParts) != int(numParts) {
+ t.Errorf("Number of unsent parts does not match original number of "+
+ "parts when none have been sent.\nexpected: %d\nreceived: %d",
+ numParts, len(unsentParts))
+ }
+
+ // Ensure all parts have the proper part number
+ for i, p := range unsentParts {
+ if int(p.partNum) != i {
+ t.Errorf("Part has incorrect part number."+
+ "\nexpected: %d\nreceived: %d", i, p.partNum)
+ }
+ }
+
+ // Use every other part
+ for i := range unsentParts {
+ if i%2 == 0 {
+ unsentParts[i].MarkArrived()
+ }
+ }
+
+ // Check that only have the number of parts is returned
+ unsentParts = st.GetUnsentParts()
+ if len(unsentParts) != int(numParts)/2 {
+ t.Errorf("Number of unsent parts is not half original number after "+
+ "half have been marked as arrived.\nexpected: %d\nreceived: %d",
+ numParts/2, len(unsentParts))
+ }
+
+ // Ensure all parts have the proper part number
+ for i, p := range unsentParts {
+ if int(p.partNum) != i*2+1 {
+ t.Errorf("Part has incorrect part number."+
+ "\nexpected: %d\nreceived: %d", i*2+1, p.partNum)
+ }
+ }
+
+ // Use the rest of the parts
+ for i := range unsentParts {
+ unsentParts[i].MarkArrived()
+ }
+
+ // Check that no sent parts are returned
+ unsentParts = st.GetUnsentParts()
+ if len(unsentParts) != 0 {
+ t.Errorf("Number of unsent parts is not zero after all have been "+
+ "marked as arrived.\nexpected: %d\nreceived: %d",
+ 0, len(unsentParts))
+ }
+}
+
+// Tests that SentTransfer.getPartData returns all the correct parts at their
+// expected indexes.
+func TestSentTransfer_getPartData(t *testing.T) {
+ st, parts, _, _, _ := newTestSentTransfer(16, t)
+
+ for i, part := range parts {
+ partData := st.getPartData(uint16(i))
+
+ if !bytes.Equal(part, partData) {
+ t.Errorf("Incorrect part #%d.\nexpected: %q\nreceived: %q",
+ i, part, partData)
+ }
+ }
+}
+
+// Tests that SentTransfer.getPartData panics when the part number is not within
+// the range of part numbers.
+func TestSentTransfer_getPartData_OutOfRangePanic(t *testing.T) {
+ st, parts, _, _, _ := newTestSentTransfer(16, t)
+
+ invalidPartNum := uint16(len(parts) + 1)
+ expectedErr := fmt.Sprintf(errNoPartNum, invalidPartNum, st.tid, st.fileName)
+
+ defer func() {
+ r := recover()
+ if r == nil || r != expectedErr {
+ t.Errorf("getPartData did not return the expected error when the "+
+ "part number %d is out of range.\nexpected: %s\nreceived: %+v",
+ invalidPartNum, expectedErr, r)
+ }
+ }()
+
+ _ = st.getPartData(invalidPartNum)
+}
+
+// Tests that after setting all parts as arrived via SentTransfer.markArrived,
+// there are no unsent parts left and the transfer is marked as Completed.
+func TestSentTransfer_markArrived(t *testing.T) {
+ st, parts, _, _, _ := newTestSentTransfer(16, t)
+
+ // Mark all parts as arrived
+ for i := range parts {
+ st.markArrived(uint16(i))
+ }
+
+ // Check that all parts are marked as arrived
+ unsentParts := st.GetUnsentParts()
+ if len(unsentParts) != 0 {
+ t.Errorf("There are %d unsent parts.", len(unsentParts))
+ }
+
+ if st.status != Completed {
+ t.Errorf("Status not correctly marked.\nexpected: %s\nreceived: %s",
+ Completed, st.status)
+ }
+}
+
+// Tests that SentTransfer.markTransferFailed changes the status of the transfer
+// to Failed.
+func TestSentTransfer_markTransferFailed(t *testing.T) {
+ st, _, _, _, _ := newTestSentTransfer(16, t)
+
+ st.markTransferFailed()
+
+ if st.status != Failed {
+ t.Errorf("Status not correctly marked.\nexpected: %s\nreceived: %s",
+ Failed, st.status)
+ }
+}
+
+// Tests that SentTransfer.Status returns the correct status of the transfer.
+func TestSentTransfer_Status(t *testing.T) {
+ st, parts, _, _, _ := newTestSentTransfer(16, t)
+
+ // Check that it is Running
+ if st.Status() != Running {
+ t.Errorf("Status returned incorrect status.\nexpected: %s\nreceived: %s",
+ Running, st.Status())
+ }
+
+ // Mark all parts as arrived
+ for i := range parts {
+ st.markArrived(uint16(i))
+ }
+
+ // Check that it is Completed
+ if st.Status() != Completed {
+ t.Errorf("Status returned incorrect status.\nexpected: %s\nreceived: %s",
+ Completed, st.Status())
+ }
+
+ // Mark transfer failed
+ st.markTransferFailed()
+
+ // Check that it is Failed
+ if st.Status() != Failed {
+ t.Errorf("Status returned incorrect status.\nexpected: %s\nreceived: %s",
+ Failed, st.Status())
+ }
+}
+
+// Tests that SentTransfer.NumParts returns the correct number of parts.
+func TestSentTransfer_NumParts(t *testing.T) {
+ numParts := uint16(16)
+ st, _, _, _, _ := newTestSentTransfer(numParts, t)
+
+ if st.NumParts() != numParts {
+ t.Errorf("Incorrect number of parts.\nexpected: %d\nreceived: %d",
+ numParts, st.NumParts())
+ }
+}
+
+// Tests that SentTransfer.TransferID returns the correct transfer ID.
+func TestSentTransfer_TransferID(t *testing.T) {
+ st, _, _, _, _ := newTestSentTransfer(16, t)
+
+ if st.TransferID() != st.tid {
+ t.Errorf("Incorrect transfer ID.\nexpected: %s\nreceived: %s",
+ st.tid, st.TransferID())
+ }
+}
+
+// Tests that SentTransfer.FileName returns the correct file name.
+func TestSentTransfer_FileName(t *testing.T) {
+ st, _, _, _, _ := newTestSentTransfer(16, t)
+
+ if st.FileName() != st.fileName {
+ t.Errorf("Incorrect transfer ID.\nexpected: %s\nreceived: %s",
+ st.fileName, st.FileName())
+ }
+}
+
+// Tests that SentTransfer.Recipient returns the correct recipient ID.
+func TestSentTransfer_Recipient(t *testing.T) {
+ st, _, _, _, _ := newTestSentTransfer(16, t)
+
+ if !st.Recipient().Cmp(st.recipient) {
+ t.Errorf("Incorrect recipient ID.\nexpected: %s\nreceived: %s",
+ st.recipient, st.Recipient())
+ }
+}
+
+// Tests that SentTransfer.NumArrived returns the correct number of arrived
+// parts.
+func TestSentTransfer_NumArrived(t *testing.T) {
+ st, parts, _, _, _ := newTestSentTransfer(16, t)
+
+ if st.NumArrived() != 0 {
+ t.Errorf("Incorrect number of arrived parts."+
+ "\nexpected: %d\nreceived: %d", 0, st.NumArrived())
+ }
+
+ // Mark all parts as arrived
+ for i := range parts {
+ st.markArrived(uint16(i))
+ }
+
+ if uint32(st.NumArrived()) != st.partStatus.GetNumKeys() {
+ t.Errorf("Incorrect number of arrived parts."+
+ "\nexpected: %d\nreceived: %d",
+ uint32(st.NumArrived()), st.partStatus.GetNumKeys())
+ }
+}
+
+// Tests that the state vector returned by SentTransfer.CopyPartStatusVector
+// has the same values as the original but is a copy.
+func TestSentTransfer_CopyPartStatusVector(t *testing.T) {
+ st, _, _, _, _ := newTestSentTransfer(16, t)
+
+ // Check that the vectors have the same unused parts
+ partStatus := st.CopyPartStatusVector()
+ if !reflect.DeepEqual(
+ partStatus.GetUnusedKeyNums(), st.partStatus.GetUnusedKeyNums()) {
+ t.Errorf("Copied part status does not match original."+
+ "\nexpected: %v\nreceived: %v",
+ st.partStatus.GetUnusedKeyNums(), partStatus.GetUnusedKeyNums())
+ }
+
+ // Modify the state
+ st.markArrived(5)
+
+ // Check that the copied state is different
+ if reflect.DeepEqual(
+ partStatus.GetUnusedKeyNums(), st.partStatus.GetUnusedKeyNums()) {
+ t.Errorf("Old copied part status matches new status."+
+ "\nexpected: %v\nreceived: %v",
+ st.partStatus.GetUnusedKeyNums(), partStatus.GetUnusedKeyNums())
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Storage Functions //
+////////////////////////////////////////////////////////////////////////////////
+
+// Tests that a SentTransfer loaded via loadSentTransfer matches the original.
+func Test_loadSentTransfer(t *testing.T) {
+ st, _, _, _, kv := newTestSentTransfer(64, t)
+
+ loadedSt, err := loadSentTransfer(st.tid, kv)
+ if err != nil {
+ t.Errorf("Failed to load SentTransfer: %+v", err)
+ }
+
+ if !reflect.DeepEqual(st, loadedSt) {
+ t.Errorf("Loaded SentTransfer does not match original."+
+ "\nexpected: %+v\nreceived: %+v", st, loadedSt)
+ }
+}
+
+// Tests that SentTransfer.Delete deletes the storage backend of the
+// SentTransfer and that it cannot be loaded again.
+func TestSentTransfer_Delete(t *testing.T) {
+ st, _, _, _, kv := newTestSentTransfer(64, t)
+
+ err := st.Delete()
+ if err != nil {
+ t.Errorf("Delete returned an error: %+v", err)
+ }
+
+ _, err = loadSentTransfer(st.tid, kv)
+ if err == nil {
+ t.Errorf("Loaded sent transfer that was deleted.")
+ }
+}
+
+// Tests that the fields saved by SentTransfer.save can be loaded from storage.
+func TestSentTransfer_save(t *testing.T) {
+ st, _, _, _, _ := newTestSentTransfer(64, t)
+
+ err := st.save()
+ if err != nil {
+ t.Errorf("save returned an error: %+v", err)
+ }
+
+ _, err = st.kv.Get(sentTransferStoreKey, sentTransferStoreVersion)
+ if err != nil {
+ t.Errorf("Failed to load saved SentTransfer: %+v", err)
+ }
+}
+
+// Tests that a SentTransfer marshalled via SentTransfer.marshal and
+// unmarshalled via unmarshalSentTransfer matches the original.
+func TestSentTransfer_marshal_unmarshalSentTransfer(t *testing.T) {
+ st := &SentTransfer{
+ fileName: "transferName",
+ recipient: id.NewIdFromString("user", id.User, t),
+ status: Failed,
+ parts: [][]byte{[]byte("Message"), []byte("Part")},
+ }
+
+ data, err := st.marshal()
+ if err != nil {
+ t.Errorf("marshal returned an error: %+v", err)
+ }
+
+ fileName, recipient, status, parts, err := unmarshalSentTransfer(data)
+ if err != nil {
+ t.Errorf("Failed to unmarshal SentTransfer: %+v", err)
+ }
+
+ if st.fileName != fileName {
+ t.Errorf("Incorrect file name.\nexpected: %q\nreceived: %q",
+ st.fileName, fileName)
+ }
+
+ if !st.recipient.Cmp(recipient) {
+ t.Errorf("Incorrect recipient.\nexpected: %s\nreceived: %s",
+ st.recipient, recipient)
+ }
+
+ if status != status {
+ t.Errorf("Incorrect status.\nexpected: %s\nreceived: %s",
+ status, status)
+ }
+
+ if !reflect.DeepEqual(st.parts, parts) {
+ t.Errorf("Incorrect parts.\nexpected: %q\nreceived: %q",
+ st.parts, parts)
+ }
+}
+
+// Consistency test of makeSentTransferPrefix.
+func Test_makeSentTransferPrefix_Consistency(t *testing.T) {
+ expectedPrefixes := []string{
+ "SentFileTransferStore/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "SentFileTransferStore/AQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "SentFileTransferStore/AgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "SentFileTransferStore/AwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "SentFileTransferStore/BAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "SentFileTransferStore/BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "SentFileTransferStore/BgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "SentFileTransferStore/BwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "SentFileTransferStore/CAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ "SentFileTransferStore/CQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
+ }
+
+ for i, expected := range expectedPrefixes {
+ tid := ftCrypto.TransferID{byte(i)}
+ prefix := makeSentTransferPrefix(&tid)
+
+ if expected != prefix {
+ t.Errorf("Prefix #%d does not match expected."+
+ "\nexpected: %q\nreceived: %q", i, expected, prefix)
+ }
+ }
+}
+
+const numPrimeBytes = 512
+
+// newTestSentTransfer creates a new SentTransfer for testing.
+func newTestSentTransfer(numParts uint16, t *testing.T) (
+ *SentTransfer, [][]byte, *ftCrypto.TransferKey, uint16, *versioned.KV) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ recipient := id.NewIdFromString("recipient", id.User, t)
+ key, _ := ftCrypto.NewTransferKey(csprng.NewSystemRNG())
+ tid, _ := ftCrypto.NewTransferID(csprng.NewSystemRNG())
+ numFps := 2 * numParts
+ fileName := "helloFile"
+ parts := generateTestParts(numParts)
+
+ st, err := newSentTransfer(recipient, &key, &tid, fileName, parts, numFps, kv)
+ if err != nil {
+ t.Errorf("Failed to make new SentTransfer: %+v", err)
+ }
+
+ return st, parts, &key, numFps, kv
+}
+
+// generateTestParts generates a list of file parts of the correct size to be
+// encrypted/decrypted.
+func generateTestParts(numParts uint16) [][]byte {
+ // Calculate part size
+ partSize := fileMessage.NewPartMessage(
+ format.NewMessage(numPrimeBytes).ContentsSize()).GetPartSize()
+
+ // Create list of parts and fill
+ parts := make([][]byte, numParts)
+ for i := range parts {
+ parts[i] = make([]byte, partSize)
+ copy(parts[i], "Hello "+strconv.Itoa(i))
+ }
+
+ return parts
+}
diff --git a/fileTransfer2/store/sent_test.go b/fileTransfer2/store/sent_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..e07357f2fd97191cced221a3e44209864293c23f
--- /dev/null
+++ b/fileTransfer2/store/sent_test.go
@@ -0,0 +1,279 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package store
+
+import (
+ "bytes"
+ "fmt"
+ "gitlab.com/elixxir/client/storage/versioned"
+ ftCrypto "gitlab.com/elixxir/crypto/fileTransfer"
+ "gitlab.com/elixxir/ekv"
+ "gitlab.com/xx_network/crypto/csprng"
+ "gitlab.com/xx_network/primitives/id"
+ "reflect"
+ "sort"
+ "strconv"
+ "testing"
+)
+
+// Tests that NewOrLoadSent returns a new Sent when none exist in storage and
+// that the list of unsent parts is nil.
+func TestNewOrLoadSent_New(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ expected := &Sent{
+ transfers: make(map[ftCrypto.TransferID]*SentTransfer),
+ kv: kv.Prefix(sentTransfersStorePrefix),
+ }
+
+ s, unsentParts, err := NewOrLoadSent(kv)
+ if err != nil {
+ t.Errorf("NewOrLoadSent returned an error: %+v", err)
+ }
+
+ if !reflect.DeepEqual(expected, s) {
+ t.Errorf("New Sent does not match expected."+
+ "\nexpected: %+v\nreceived: %+v", expected, s)
+ }
+
+ if unsentParts != nil {
+ t.Errorf("List of parts should be nil when not loading: %+v",
+ unsentParts)
+ }
+}
+
+// Tests that NewOrLoadSent returns a loaded Sent when one exist in storage and
+// that the list of unsent parts is correct.
+func TestNewOrLoadSent_Load(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ s, _, err := NewOrLoadSent(kv)
+ if err != nil {
+ t.Errorf("Failed to make new Sent: %+v", err)
+ }
+ var expectedUnsentParts []Part
+
+ // Create and add transfers to map and save
+ for i := 0; i < 10; i++ {
+ key, _ := ftCrypto.NewTransferKey(csprng.NewSystemRNG())
+ tid, _ := ftCrypto.NewTransferID(csprng.NewSystemRNG())
+ st, err2 := s.AddTransfer(
+ id.NewIdFromString("recipient"+strconv.Itoa(i), id.User, t),
+ &key, &tid, "file"+strconv.Itoa(i),
+ generateTestParts(uint16(10+i)), uint16(2*(10+i)))
+ if err2 != nil {
+ t.Errorf("Failed to add transfer #%d: %+v", i, err2)
+ }
+ expectedUnsentParts = append(expectedUnsentParts, st.GetUnsentParts()...)
+ }
+ if err = s.save(); err != nil {
+ t.Errorf("Failed to make save filled Sent: %+v", err)
+ }
+
+ // Load Sent
+ loadedSent, unsentParts, err := NewOrLoadSent(kv)
+ if err != nil {
+ t.Errorf("Failed to load Sent: %+v", err)
+ }
+
+ // Check that the loaded Sent matches original
+ if !reflect.DeepEqual(s, loadedSent) {
+ t.Errorf("Loaded Sent does not match original."+
+ "\nexpected: %v\nreceived: %v", s, loadedSent)
+ }
+
+ sort.Slice(unsentParts, func(i, j int) bool {
+ switch bytes.Compare(unsentParts[i].TransferID()[:],
+ unsentParts[j].TransferID()[:]) {
+ case -1:
+ return true
+ case 1:
+ return false
+ default:
+ return unsentParts[i].partNum < unsentParts[j].partNum
+ }
+ })
+
+ sort.Slice(expectedUnsentParts, func(i, j int) bool {
+ switch bytes.Compare(expectedUnsentParts[i].TransferID()[:],
+ expectedUnsentParts[j].TransferID()[:]) {
+ case -1:
+ return true
+ case 1:
+ return false
+ default:
+ return expectedUnsentParts[i].partNum < expectedUnsentParts[j].partNum
+ }
+ })
+
+ // Check that the unsent parts matches expected
+ if !reflect.DeepEqual(expectedUnsentParts, unsentParts) {
+ t.Errorf("Incorrect unsent parts.\nexpected: %v\nreceived: %v",
+ expectedUnsentParts, unsentParts)
+ }
+}
+
+// Tests that Sent.AddTransfer makes a new transfer and adds it to the list.
+func TestSent_AddTransfer(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ s, _, _ := NewOrLoadSent(kv)
+
+ key, _ := ftCrypto.NewTransferKey(csprng.NewSystemRNG())
+ tid, _ := ftCrypto.NewTransferID(csprng.NewSystemRNG())
+
+ st, err := s.AddTransfer(id.NewIdFromString("recipient", id.User, t),
+ &key, &tid, "file", generateTestParts(10), 20)
+ if err != nil {
+ t.Errorf("Failed to add new transfer: %+v", err)
+ }
+
+ // Check that the transfer was added
+ if _, exists := s.transfers[*st.tid]; !exists {
+ t.Errorf("No transfer with ID %s exists.", st.tid)
+ }
+}
+
+// Tests that Sent.AddTransfer returns an error when adding a transfer ID that
+// already exists.
+func TestSent_AddTransfer_TransferAlreadyExists(t *testing.T) {
+ tid := ftCrypto.TransferID{0}
+ s := &Sent{
+ transfers: map[ftCrypto.TransferID]*SentTransfer{tid: nil},
+ }
+
+ expectedErr := fmt.Sprintf(errAddExistingSentTransfer, tid)
+ _, err := s.AddTransfer(nil, nil, &tid, "", nil, 0)
+ if err == nil || err.Error() != expectedErr {
+ t.Errorf("Received unexpected error when adding transfer that already "+
+ "exists.\nexpected: %s\nreceived: %+v", expectedErr, err)
+ }
+}
+
+// Tests that Sent.GetTransfer returns the expected transfer.
+func TestSent_GetTransfer(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ s, _, _ := NewOrLoadSent(kv)
+
+ key, _ := ftCrypto.NewTransferKey(csprng.NewSystemRNG())
+ tid, _ := ftCrypto.NewTransferID(csprng.NewSystemRNG())
+
+ st, err := s.AddTransfer(id.NewIdFromString("recipient", id.User, t),
+ &key, &tid, "file", generateTestParts(10), 20)
+ if err != nil {
+ t.Errorf("Failed to add new transfer: %+v", err)
+ }
+
+ // Check that the transfer was added
+ receivedSt, exists := s.GetTransfer(st.tid)
+ if !exists {
+ t.Errorf("No transfer with ID %s exists.", st.tid)
+ }
+
+ if !reflect.DeepEqual(st, receivedSt) {
+ t.Errorf("Received SentTransfer does not match expected."+
+ "\nexpected: %+v\nreceived: %+v", st, receivedSt)
+ }
+}
+
+// Tests that Sent.RemoveTransfer removes the transfer from the list.
+func TestSent_RemoveTransfer(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ s, _, _ := NewOrLoadSent(kv)
+
+ key, _ := ftCrypto.NewTransferKey(csprng.NewSystemRNG())
+ tid, _ := ftCrypto.NewTransferID(csprng.NewSystemRNG())
+
+ st, err := s.AddTransfer(id.NewIdFromString("recipient", id.User, t),
+ &key, &tid, "file", generateTestParts(10), 20)
+ if err != nil {
+ t.Errorf("Failed to add new transfer: %+v", err)
+ }
+
+ // Delete the transfer
+ err = s.RemoveTransfer(st.tid)
+ if err != nil {
+ t.Errorf("RemoveTransfer returned an error: %+v", err)
+ }
+
+ // Check that the transfer was deleted
+ _, exists := s.GetTransfer(st.tid)
+ if exists {
+ t.Errorf("Transfer %s exists.", st.tid)
+ }
+
+ // Remove transfer that was already removed
+ err = s.RemoveTransfer(st.tid)
+ if err != nil {
+ t.Errorf("RemoveTransfer returned an error: %+v", err)
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Storage Functions //
+////////////////////////////////////////////////////////////////////////////////
+
+// Tests that Sent.save saves the transfer ID list to storage by trying to load
+// it after a save.
+func TestSent_save(t *testing.T) {
+ kv := versioned.NewKV(ekv.MakeMemstore())
+ s, _, _ := NewOrLoadSent(kv)
+ s.transfers = map[ftCrypto.TransferID]*SentTransfer{
+ ftCrypto.TransferID{0}: nil, ftCrypto.TransferID{1}: nil,
+ ftCrypto.TransferID{2}: nil, ftCrypto.TransferID{3}: nil,
+ }
+
+ err := s.save()
+ if err != nil {
+ t.Errorf("Failed to save transfer ID list: %+v", err)
+ }
+
+ _, err = s.kv.Get(sentTransfersStoreKey, sentTransfersStoreVersion)
+ if err != nil {
+ t.Errorf("Failed to load transfer ID list: %+v", err)
+ }
+}
+
+// Tests that the transfer IDs keys in the map marshalled by
+// marshalSentTransfersMap and unmarshalled by unmarshalTransferIdList match the
+// original.
+func Test_marshalSentTransfersMap_unmarshalTransferIdList(t *testing.T) {
+ // Build map of transfer IDs
+ transfers := make(map[ftCrypto.TransferID]*SentTransfer, 10)
+ for i := 0; i < 10; i++ {
+ tid, _ := ftCrypto.NewTransferID(csprng.NewSystemRNG())
+ transfers[tid] = nil
+ }
+
+ data, err := marshalSentTransfersMap(transfers)
+ if err != nil {
+ t.Errorf("marshalSentTransfersMap returned an error: %+v", err)
+ }
+
+ tidList, err := unmarshalTransferIdList(data)
+ if err != nil {
+ t.Errorf("unmarshalSentTransfer returned an error: %+v", err)
+ }
+
+ for _, tid := range tidList {
+ if _, exists := transfers[tid]; exists {
+ delete(transfers, tid)
+ } else {
+ t.Errorf("Transfer %s does not exist in list.", tid)
+ }
+ }
+
+ if len(transfers) != 0 {
+ t.Errorf("%d transfers not in unmarshalled list: %v",
+ len(transfers), transfers)
+ }
+}
diff --git a/fileTransfer2/store/transferStatus.go b/fileTransfer2/store/transferStatus.go
new file mode 100644
index 0000000000000000000000000000000000000000..e8674b711f36a62f064548f1f9928598cf44aa8d
--- /dev/null
+++ b/fileTransfer2/store/transferStatus.go
@@ -0,0 +1,64 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package store
+
+import (
+ "encoding/binary"
+ "strconv"
+)
+
+// TransferStatus indicates the state of the transfer.
+type TransferStatus int
+
+const (
+ // Running indicates that the transfer is in the processes of sending
+ Running TransferStatus = iota
+
+ // Completed indicates that all file parts have been sent and arrived
+ Completed
+
+ // Failed indicates that the transfer has run out of sending retries
+ Failed
+)
+
+const invalidTransferStatusStringErr = "INVALID TransferStatus: "
+
+// String prints the string representation of the TransferStatus. This function
+// satisfies the fmt.Stringer interface.
+func (ts TransferStatus) String() string {
+ switch ts {
+ case Running:
+ return "running"
+ case Completed:
+ return "completed"
+ case Failed:
+ return "failed"
+ default:
+ return invalidTransferStatusStringErr + strconv.Itoa(int(ts))
+ }
+}
+
+// Marshal returns the byte representation of the TransferStatus.
+func (ts TransferStatus) Marshal() []byte {
+ b := make([]byte, 8)
+ binary.LittleEndian.PutUint64(b, uint64(ts))
+ return b
+}
+
+// UnmarshalTransferStatus unmarshalls the 8-byte byte slice into a
+// TransferStatus.
+func UnmarshalTransferStatus(b []byte) TransferStatus {
+ return TransferStatus(binary.LittleEndian.Uint64(b))
+}
diff --git a/fileTransfer2/store/transferStatus_test.go b/fileTransfer2/store/transferStatus_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..f87c666c5b5dbec24db6149314b7f339e88ce13c
--- /dev/null
+++ b/fileTransfer2/store/transferStatus_test.go
@@ -0,0 +1,54 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package store
+
+import (
+ "strconv"
+ "testing"
+)
+
+// Tests that TransferStatus.String returns the expected string for each value
+// of TransferStatus.
+func Test_TransferStatus_String(t *testing.T) {
+ testValues := map[TransferStatus]string{
+ Running: "running",
+ Completed: "completed",
+ Failed: "failed",
+ 100: invalidTransferStatusStringErr + strconv.Itoa(100),
+ }
+
+ for status, expected := range testValues {
+ if expected != status.String() {
+ t.Errorf("TransferStatus string incorrect."+
+ "\nexpected: %s\nreceived: %s", expected, status.String())
+ }
+ }
+}
+
+// Tests that a marshalled and unmarshalled TransferStatus matches the original.
+func Test_TransferStatus_Marshal_UnmarshalTransferStatus(t *testing.T) {
+ testValues := []TransferStatus{Running, Completed, Failed}
+
+ for _, status := range testValues {
+ marshalledStatus := status.Marshal()
+
+ newStatus := UnmarshalTransferStatus(marshalledStatus)
+
+ if status != newStatus {
+ t.Errorf("Marshalled and unmarshalled TransferStatus does not "+
+ "match original.\nexpected: %s\nreceived: %s", status, newStatus)
+ }
+ }
+}
diff --git a/fileTransfer2/utils_test.go b/fileTransfer2/utils_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..8f1c448923884d8cec9aeff4f48f3abb061516d2
--- /dev/null
+++ b/fileTransfer2/utils_test.go
@@ -0,0 +1,176 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2020 xx network SEZC //
+// //
+// Use of this source code is governed by a license that can be found in the //
+// LICENSE file //
+////////////////////////////////////////////////////////////////////////////////
+
+package fileTransfer2
+
+import (
+ "bytes"
+ "encoding/binary"
+ jww "github.com/spf13/jwalterweatherman"
+ "gitlab.com/elixxir/client/cmix"
+ "gitlab.com/elixxir/client/cmix/identity/receptionID"
+ "gitlab.com/elixxir/client/cmix/message"
+ "gitlab.com/elixxir/client/cmix/rounds"
+ "gitlab.com/elixxir/primitives/format"
+ "gitlab.com/xx_network/primitives/id"
+ "gitlab.com/xx_network/primitives/id/ephemeral"
+ "io"
+ "math/rand"
+ "sync"
+ "testing"
+ "time"
+)
+
+// newFile generates a file with random data of size numParts * partSize.
+// Returns the full file and the file parts. If the partSize allows, each part
+// starts with a "|<[PART_001]" and ends with a ">|".
+func newFile(numParts uint16, partSize int, prng io.Reader, t *testing.T) (
+ []byte, [][]byte) {
+ const (
+ prefix = "|<[PART_%3d]"
+ suffix = ">|"
+ )
+ // Create file buffer of the expected size
+ fileBuff := bytes.NewBuffer(make([]byte, 0, int(numParts)*partSize))
+ partList := make([][]byte, numParts)
+
+ // Create new rand.Rand with the seed generated from the io.Reader
+ b := make([]byte, 8)
+ _, err := prng.Read(b)
+ if err != nil {
+ t.Errorf("Failed to generate random seed: %+v", err)
+ }
+ seed := binary.LittleEndian.Uint64(b)
+ randPrng := rand.New(rand.NewSource(int64(seed)))
+
+ for partNum := range partList {
+ s := RandStringBytes(partSize, randPrng)
+ if len(s) >= (len(prefix) + len(suffix)) {
+ partList[partNum] = []byte(
+ prefix + s[:len(s)-(len(prefix)+len(suffix))] + suffix)
+ } else {
+ partList[partNum] = []byte(s)
+ }
+
+ fileBuff.Write(partList[partNum])
+ }
+
+ return fileBuff.Bytes(), partList
+}
+
+const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+// RandStringBytes generates a random string of length n consisting of the
+// characters in letterBytes.
+func RandStringBytes(n int, prng *rand.Rand) string {
+ b := make([]byte, n)
+ for i := range b {
+ b[i] = letterBytes[prng.Intn(len(letterBytes))]
+ }
+ return string(b)
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Mock cMix Client //
+////////////////////////////////////////////////////////////////////////////////
+
+type mockCmixHandler struct {
+ sync.Mutex
+ processorMap map[format.Fingerprint]message.Processor
+}
+
+func newMockCmixHandler() *mockCmixHandler {
+ return &mockCmixHandler{
+ processorMap: make(map[format.Fingerprint]message.Processor),
+ }
+}
+
+type mockCmix struct {
+ myID *id.ID
+ numPrimeBytes int
+ health bool
+ handler *mockCmixHandler
+ healthCBs map[uint64]func(b bool)
+ healthIndex uint64
+ sync.Mutex
+}
+
+func newMockCmix(myID *id.ID, handler *mockCmixHandler) *mockCmix {
+ return &mockCmix{
+ myID: myID,
+ numPrimeBytes: 97,
+ // numPrimeBytes: 4096,
+ health: true,
+ handler: handler,
+ healthCBs: make(map[uint64]func(b bool)),
+ healthIndex: 0,
+ }
+}
+
+func (m *mockCmix) GetMaxMessageLength() int {
+ msg := format.NewMessage(m.numPrimeBytes)
+ return msg.ContentsSize()
+}
+
+func (m *mockCmix) SendMany(messages []cmix.TargetedCmixMessage,
+ _ cmix.CMIXParams) (id.Round, []ephemeral.Id, error) {
+ m.handler.Lock()
+ for _, targetedMsg := range messages {
+ msg := format.NewMessage(m.numPrimeBytes)
+ msg.SetContents(targetedMsg.Payload)
+ msg.SetMac(targetedMsg.Mac)
+ msg.SetKeyFP(targetedMsg.Fingerprint)
+ m.handler.processorMap[targetedMsg.Fingerprint].Process(msg,
+ receptionID.EphemeralIdentity{Source: targetedMsg.Recipient},
+ rounds.Round{ID: 42})
+ }
+ m.handler.Unlock()
+ return 42, []ephemeral.Id{}, nil
+}
+
+func (m *mockCmix) AddFingerprint(_ *id.ID, fp format.Fingerprint, mp message.Processor) error {
+ m.Lock()
+ defer m.Unlock()
+ m.handler.processorMap[fp] = mp
+ return nil
+}
+
+func (m *mockCmix) DeleteFingerprint(_ *id.ID, fp format.Fingerprint) {
+ m.handler.Lock()
+ delete(m.handler.processorMap, fp)
+ m.handler.Unlock()
+}
+
+func (m *mockCmix) IsHealthy() bool {
+ return m.health
+}
+
+func (m *mockCmix) WasHealthy() bool { return true }
+
+func (m *mockCmix) AddHealthCallback(f func(bool)) uint64 {
+ m.Lock()
+ defer m.Unlock()
+ m.healthIndex++
+ m.healthCBs[m.healthIndex] = f
+ go f(true)
+ return m.healthIndex
+}
+
+func (m *mockCmix) RemoveHealthCallback(healthID uint64) {
+ m.Lock()
+ defer m.Unlock()
+ if _, exists := m.healthCBs[healthID]; !exists {
+ jww.FATAL.Panicf("No health callback with ID %d exists.", healthID)
+ }
+ delete(m.healthCBs, healthID)
+}
+
+func (m *mockCmix) GetRoundResults(_ time.Duration,
+ roundCallback cmix.RoundEventCallback, _ ...id.Round) error {
+ go roundCallback(true, false, map[id.Round]cmix.RoundResult{42: {}})
+ return nil
+}