Extended Round Storage initial implementation

go.sum

@@ -62,8 +62,6 @@ gitlab.com/elixxir/crypto v0.0.0-20200707005343-97f868cbd930 h1:9qzfwyR12OYgn3j3
 gitlab.com/elixxir/crypto v0.0.0-20200707005343-97f868cbd930/go.mod h1:LHBAaEf48a0/AjU118rjoworH0LgXifhAqmNX3ZRvME=
 gitlab.com/elixxir/primitives v0.0.0-20200706165052-9fe7a4fb99a3 h1:GTfflZBNLeBq3UApYog0J3+hytdkoRsDduGQji2wyEU=
 gitlab.com/elixxir/primitives v0.0.0-20200706165052-9fe7a4fb99a3/go.mod h1:OQgUZq7SjnE0b+8+iIAT2eqQF+2IFHn73tOo+aV11mg=
-gitlab.com/xx_network/comms v0.0.0-20200709165104-1fcde4b1729d h1:Vpg93y1f3AzFHfZFqMjbrNpRSfUCjmOg0rMLqY0venE=
-gitlab.com/xx_network/comms v0.0.0-20200709165104-1fcde4b1729d/go.mod h1:CX2wQaDwnnk68etjJzIzyJ9Qfxl01KuTKKLpgXRhIYY=
 gitlab.com/xx_network/comms v0.0.0-20200721184230-3e4aa5dce2db h1:ZT35+F8s8nFPPEDBWxe9U1MDhj/NE5a512tUHnW4gXE=
 gitlab.com/xx_network/comms v0.0.0-20200721184230-3e4aa5dce2db/go.mod h1:CX2wQaDwnnk68etjJzIzyJ9Qfxl01KuTKKLpgXRhIYY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

network/dataStructures/extendedRoundStorage.go

+package dataStructures
+
+import (
+	pb "gitlab.com/elixxir/comms/mixmessages"
+	"gitlab.com/elixxir/primitives/id"
+)
+
+type ExternalRoundStorage interface {
+	// Store: stores the round info inside the underlying storage medium, which generally is a database. Store will
+	// add the round info to the database if it doesn't exist and will only overwrite the data if it does exist in the
+	// event that the update ID of the passed in data is greater than the update ID of the existing round info.
+	Store(*pb.RoundInfo) error
+	// Retrieve will return the round info for the given round ID and will return nil but not an error if it does not
+	// exist.
+	Retrieve(id id.Round) (*pb.RoundInfo, error)
+	// RetrieveMany will return all rounds passed in in the ID list, if the round doesn't its reciprocal entry in the
+	// returned slice will be blank.
+	RetrieveMany(rounds []id.Round) ([]*pb.RoundInfo, error)
+	// RetrieveRange will return all rounds in the range, if the round doesn't exist the reciprocal entry in the
+	// returned slice will be blank.
+	RetrieveRange(first, last id.Round) ([]*pb.RoundInfo, error)
+}

network/historicalRoundData.go

+package network
+
+import (
+	"errors"
+	pb "gitlab.com/elixxir/comms/mixmessages"
+	"gitlab.com/elixxir/primitives/id"
+)
+
+func (i *Instance) GetHistoricalRound(id id.Round) (*pb.RoundInfo, error) {
+	if i.ers != nil {
+		return i.ers.Retrieve(id)
+	}
+	return nil, errors.New("no ExternalRoundStorage object was defined on instance creation")
+}
+
+func (i *Instance) GetHistoricalRounds(rounds []id.Round) ([]*pb.RoundInfo, error) {
+	if i.ers != nil {
+		return i.ers.RetrieveMany(rounds)
+	}
+	return nil, errors.New("no ExternalRoundStorage object was defined on instance creation")
+}
+
+func (i *Instance) GetHistoricalRoundRange(first, last id.Round) ([]*pb.RoundInfo, error) {
+	if i.ers != nil {
+		return i.ers.RetrieveRange(first, last)
+	}
+	return nil, errors.New("no ExternalRoundStorage object was defined on instance creation")
+}

network/historicalRoundData_test.go

+package network
+
+import (
+	jww "github.com/spf13/jwalterweatherman"
+	pb "gitlab.com/elixxir/comms/mixmessages"
+	ds "gitlab.com/elixxir/comms/network/dataStructures"
+	"gitlab.com/elixxir/primitives/id"
+	"testing"
+)
+
+//region ERS Memory Map Impl
+// Memory map based ExtendedRoundStorage database
+type ersMemMap struct {
+	rounds map[id.Round]*pb.RoundInfo
+}
+
+// Store a new round info object into the map
+func (ersm ersMemMap) Store(ri *pb.RoundInfo) error {
+	rid := id.Round(ri.GetID())
+	// See if the round exists, if it does we need to check that the update ID is newer than the current
+	ori, err := ersm.Retrieve(rid)
+	if err != nil {
+		return err
+	}
+
+	if ori == nil || ori.UpdateID < ri.UpdateID {
+		ersm.rounds[rid] = ri
+	} else {
+		jww.WARN.Printf("Passed in round update ID of %v lower than currently stored ID %v",
+			ri.UpdateID, ori.UpdateID)
+	}
+
+	return nil
+}
+
+// Get a round info object from the memory map database
+func (ersm ersMemMap) Retrieve(id id.Round) (*pb.RoundInfo, error) {
+	return ersm.rounds[id], nil
+}
+
+// Get multiple specific round info objects from the memory map database
+func (ersm ersMemMap) RetrieveMany(rounds []id.Round) ([]*pb.RoundInfo, error) {
+	var r []*pb.RoundInfo
+
+	for _, round := range rounds {
+		ri, err := ersm.Retrieve(round)
+		if err != nil {
+			return nil, err
+		}
+
+		r = append(r, ri)
+	}
+
+	return r, nil
+}
+
+// Retrieve a concurrent range of round info objects from the memory map database
+func (ersm ersMemMap) RetrieveRange(first, last id.Round) ([]*pb.RoundInfo, error) {
+	idrange := uint64(last - first)
+	i := uint64(0)
+
+	var r []*pb.RoundInfo
+
+	// for some reason <= doesn't work?
+	for i < idrange+1 {
+		ri, err := ersm.Retrieve(id.Round(uint64(first) + i))
+		if err != nil {
+			return nil, err
+		}
+
+		r = append(r, ri)
+		i++
+	}
+
+	return r, nil
+}
+
+//endregion
+
+// Test we can insert a round, get it, try to update with an older ID, it doesn't update, and it does update with
+// a newer ID
+func TestERSStore(t *testing.T) {
+	// Setup
+	var ers ds.ExternalRoundStorage = ersMemMap{rounds: make(map[id.Round]*pb.RoundInfo)}
+
+	// Store a test round
+	r := pb.RoundInfo{ID: 1, UpdateID: 5}
+	err := ers.Store(&r)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	// Test that we get a new round with our info
+	ri, err := ers.Retrieve(id.Round(r.ID))
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+	if ri == nil {
+		t.Fatalf("ri object is nil, Retrieve did not return a round")
+	}
+	if ri.ID != r.ID && ri.UpdateID != r.UpdateID {
+		t.Errorf("did not return the same round ID or update ID as we inputted.")
+	}
+
+	// Update the test round
+	ru1 := pb.RoundInfo{ID: 1, UpdateID: 3}
+	err = ers.Store(&ru1)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	// Test that this updated round did not get written to the map
+	riu1, err := ers.Retrieve(id.Round(r.ID))
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+	if riu1 == nil {
+		t.Fatalf("ri object is nil, Retrieve did not return a round")
+	}
+	if riu1.UpdateID == ru1.UpdateID {
+		t.Errorf("stored round info was updated to have lower update ID of %v", riu1.UpdateID)
+	}
+
+	// Update the test round
+	ru2 := pb.RoundInfo{ID: 1, UpdateID: 10}
+	err = ers.Store(&ru2)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	// Test that this updated round did not get written to the map
+	riu2, err := ers.Retrieve(id.Round(r.ID))
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+	if riu2 == nil {
+		t.Fatalf("ri object is nil, Retrieve did not return a round")
+	}
+	if riu2.UpdateID != ru2.UpdateID {
+		t.Errorf("stored round info was not updated to have update ID of %v", riu2.UpdateID)
+	}
+}
+
+// Test that Retrieve has expected behaviour if an item doesn't exist
+func TestERSRetrieve(t *testing.T) {
+	var ers ds.ExternalRoundStorage = ersMemMap{rounds: make(map[id.Round]*pb.RoundInfo)}
+	ri, err := ers.Retrieve(id.Round(1))
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+	if ri != nil {
+		t.Errorf("returned round info was not nil")
+	}
+
+	// Store a test round
+	r := pb.RoundInfo{ID: 1, UpdateID: 5}
+	err = ers.Store(&r)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	nri, err := ers.Retrieve(id.Round(1))
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+	if nri == nil {
+		t.Fatalf("returned round info was not nil")
+	}
+	if nri.ID != r.ID || nri.UpdateID != r.UpdateID {
+		t.Errorf("Returned round or update ID did not match what we put in")
+	}
+}
+
+// Test that the RetrieveMany function will get rounds that are stored, while returning nil with no error for those
+// that are not
+func TestERSRetrieveMany(t *testing.T) {
+	// Setup
+	var ers ds.ExternalRoundStorage = ersMemMap{rounds: make(map[id.Round]*pb.RoundInfo)}
+
+	// Store a test round
+	origRound1 := pb.RoundInfo{ID: 1, UpdateID: 5}
+	err := ers.Store(&origRound1)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	// Store another test round
+	origRound2 := pb.RoundInfo{ID: 8, UpdateID: 3}
+	err = ers.Store(&origRound2)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	getRounds := []id.Round{id.Round(origRound1.ID), id.Round(origRound2.ID - 3), id.Round(origRound2.ID)}
+	returnRounds, err := ers.RetrieveMany(getRounds)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	if returnRounds[0] == nil || returnRounds[2] == nil {
+		t.Fatalf("RetrieveMany did not return a round we expected to get returned")
+	}
+	if returnRounds[1] != nil {
+		t.Errorf("Middle fake round did return a round info object")
+	}
+	if returnRounds[0].ID != origRound1.ID || returnRounds[0].UpdateID != origRound1.UpdateID {
+		t.Errorf("First returned round and original mismatched IDs")
+	}
+	if returnRounds[2].ID != origRound2.ID || returnRounds[2].UpdateID != origRound2.UpdateID {
+		t.Errorf("Second returned round and original mismatched IDs")
+	}
+}
+
+// Test that the RetrieveRange function will get a range of rounds stored, and return nil with no error for ones that
+// are not stored
+func TestERSRetrieveRange(t *testing.T) {
+	// Setup
+	var ers ds.ExternalRoundStorage = ersMemMap{rounds: make(map[id.Round]*pb.RoundInfo)}
+
+	// Store a test round
+	origRound1 := pb.RoundInfo{ID: 1, UpdateID: 5}
+	err := ers.Store(&origRound1)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	// Store another test round
+	origRound2 := pb.RoundInfo{ID: 3, UpdateID: 3}
+	err = ers.Store(&origRound2)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	returnRounds, err := ers.RetrieveRange(1, 3)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	if returnRounds[0] == nil || returnRounds[2] == nil {
+		t.Fatalf("RetrieveMany did not return a round we expected to get returned")
+	}
+	if returnRounds[1] != nil {
+		t.Errorf("Middle fake round did return a round info object")
+	}
+	if returnRounds[0].ID != origRound1.ID || returnRounds[0].UpdateID != origRound1.UpdateID {
+		t.Errorf("First returned round and original mismatched IDs")
+	}
+	if returnRounds[2].ID != origRound2.ID || returnRounds[2].UpdateID != origRound2.UpdateID {
+		t.Errorf("Second returned round and original mismatched IDs")
+	}
+}

network/instance.go

@@ -32,12 +32,14 @@ type Instance struct {
 	full         *SecuredNdf
 	roundUpdates *ds.Updates
 	roundData    *ds.Data
+	ers          ds.ExternalRoundStorage
 
 	ipOverride *ds.IpOverrideList
 }
 
 // Initializer for instance structs from base comms and NDF
-func NewInstance(c *connect.ProtoComms, partial, full *ndf.NetworkDefinition) (*Instance, error) {
+func NewInstance(c *connect.ProtoComms, partial, full *ndf.NetworkDefinition,
+	ers ds.ExternalRoundStorage) (*Instance, error) {
 	var partialNdf *SecuredNdf
 	var fullNdf *SecuredNdf
 	var err error
@@ -100,6 +102,10 @@ func NewInstance(c *connect.ProtoComms, partial, full *ndf.NetworkDefinition) (*
 		}
 	}
 
+	if ers != nil {
+		i.ers = ers
+	}
+
 	return i, nil
 }
 
@@ -109,7 +115,7 @@ func NewInstanceTesting(c *connect.ProtoComms, partial, full *ndf.NetworkDefinit
 	if t == nil {
 		panic("This is a utility function for testing purposes only!")
 	}
-	instance, err := NewInstance(c, partial, full)
+	instance, err := NewInstance(c, partial, full, nil)
 	if err != nil {
 		return nil, errors.Errorf("Unable to create instance: %+v", err)
 	}
@@ -283,6 +289,10 @@ func (i *Instance) RoundUpdate(info *pb.RoundInfo) error {
 		return err
 	}
 
+	if i.ers != nil {
+		err = i.ers.Store(info)
+	}
+
 	return nil
 }
 

network/instance_test.go

@@ -117,7 +117,7 @@ func TestNewInstanceTesting_Error(t *testing.T) {
 
 //tests newInstance errors properly when there is no NDF
 func TestNewInstance_NilNDFs(t *testing.T) {
-	_, err := NewInstance(&connect.ProtoComms{}, nil, nil)
+	_, err := NewInstance(&connect.ProtoComms{}, nil, nil, nil)
 	if err == nil {
 		t.Errorf("Creation of NewInstance without an ndf succeded")
 	} else if !strings.Contains(err.Error(), "Cannot create a network "+
@@ -200,7 +200,7 @@ func setupComm(t *testing.T) (*Instance, *mixmessages.NDF) {
 	err = signature.Sign(f, privKey)
 
 	pc := &connect.ProtoComms{}
-	i, err := NewInstance(pc, baseNDF, baseNDF)
+	i, err := NewInstance(pc, baseNDF, baseNDF, nil)
 	if err != nil {
 		t.Error(nil)
 	}
@@ -223,7 +223,7 @@ func TestInstance_RoundUpdate(t *testing.T) {
 	privKey, err := rsa.LoadPrivateKeyFromPem(priv)
 	err = signature.Sign(msg, privKey)
 
-	i, err := NewInstance(&connect.ProtoComms{}, testutils.NDF, testutils.NDF)
+	i, err := NewInstance(&connect.ProtoComms{}, testutils.NDF, testutils.NDF, nil)
 	pub := testkeys.LoadFromPath(testkeys.GetGatewayCertPath())
 	err = i.RoundUpdate(msg)
 	if err == nil {