diff --git a/go.sum b/go.sum index aded374d60ef2ffeb38479f383073bb9f76bb04b..5a97d620f0f58fe126f4006124e9f2a2ba239c77 100644 --- a/go.sum +++ b/go.sum @@ -62,8 +62,6 @@ gitlab.com/elixxir/crypto v0.0.0-20200707005343-97f868cbd930 h1:9qzfwyR12OYgn3j3 gitlab.com/elixxir/crypto v0.0.0-20200707005343-97f868cbd930/go.mod h1:LHBAaEf48a0/AjU118rjoworH0LgXifhAqmNX3ZRvME= gitlab.com/elixxir/primitives v0.0.0-20200706165052-9fe7a4fb99a3 h1:GTfflZBNLeBq3UApYog0J3+hytdkoRsDduGQji2wyEU= gitlab.com/elixxir/primitives v0.0.0-20200706165052-9fe7a4fb99a3/go.mod h1:OQgUZq7SjnE0b+8+iIAT2eqQF+2IFHn73tOo+aV11mg= -gitlab.com/xx_network/comms v0.0.0-20200709165104-1fcde4b1729d h1:Vpg93y1f3AzFHfZFqMjbrNpRSfUCjmOg0rMLqY0venE= -gitlab.com/xx_network/comms v0.0.0-20200709165104-1fcde4b1729d/go.mod h1:CX2wQaDwnnk68etjJzIzyJ9Qfxl01KuTKKLpgXRhIYY= gitlab.com/xx_network/comms v0.0.0-20200721184230-3e4aa5dce2db h1:ZT35+F8s8nFPPEDBWxe9U1MDhj/NE5a512tUHnW4gXE= gitlab.com/xx_network/comms v0.0.0-20200721184230-3e4aa5dce2db/go.mod h1:CX2wQaDwnnk68etjJzIzyJ9Qfxl01KuTKKLpgXRhIYY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/network/dataStructures/extendedRoundStorage.go b/network/dataStructures/extendedRoundStorage.go new file mode 100644 index 0000000000000000000000000000000000000000..2d05f54f6641642b411c207bd308d515616d0a3e --- /dev/null +++ b/network/dataStructures/extendedRoundStorage.go @@ -0,0 +1,22 @@ +package dataStructures + +import ( + pb "gitlab.com/elixxir/comms/mixmessages" + "gitlab.com/elixxir/primitives/id" +) + +type ExternalRoundStorage interface { + // Store: stores the round info inside the underlying storage medium, which generally is a database. Store will + // add the round info to the database if it doesn't exist and will only overwrite the data if it does exist in the + // event that the update ID of the passed in data is greater than the update ID of the existing round info. + Store(*pb.RoundInfo) error + // Retrieve will return the round info for the given round ID and will return nil but not an error if it does not + // exist. + Retrieve(id id.Round) (*pb.RoundInfo, error) + // RetrieveMany will return all rounds passed in in the ID list, if the round doesn't its reciprocal entry in the + // returned slice will be blank. + RetrieveMany(rounds []id.Round) ([]*pb.RoundInfo, error) + // RetrieveRange will return all rounds in the range, if the round doesn't exist the reciprocal entry in the + // returned slice will be blank. + RetrieveRange(first, last id.Round) ([]*pb.RoundInfo, error) +} diff --git a/network/historicalRoundData.go b/network/historicalRoundData.go new file mode 100644 index 0000000000000000000000000000000000000000..53ac093d0551150452afe900984e0bd14f5f5e9a --- /dev/null +++ b/network/historicalRoundData.go @@ -0,0 +1,28 @@ +package network + +import ( + "errors" + pb "gitlab.com/elixxir/comms/mixmessages" + "gitlab.com/elixxir/primitives/id" +) + +func (i *Instance) GetHistoricalRound(id id.Round) (*pb.RoundInfo, error) { + if i.ers != nil { + return i.ers.Retrieve(id) + } + return nil, errors.New("no ExternalRoundStorage object was defined on instance creation") +} + +func (i *Instance) GetHistoricalRounds(rounds []id.Round) ([]*pb.RoundInfo, error) { + if i.ers != nil { + return i.ers.RetrieveMany(rounds) + } + return nil, errors.New("no ExternalRoundStorage object was defined on instance creation") +} + +func (i *Instance) GetHistoricalRoundRange(first, last id.Round) ([]*pb.RoundInfo, error) { + if i.ers != nil { + return i.ers.RetrieveRange(first, last) + } + return nil, errors.New("no ExternalRoundStorage object was defined on instance creation") +} diff --git a/network/historicalRoundData_test.go b/network/historicalRoundData_test.go new file mode 100644 index 0000000000000000000000000000000000000000..49411c274c9ff1f08583ea1418082b0fd55c5bfd --- /dev/null +++ b/network/historicalRoundData_test.go @@ -0,0 +1,251 @@ +package network + +import ( + jww "github.com/spf13/jwalterweatherman" + pb "gitlab.com/elixxir/comms/mixmessages" + ds "gitlab.com/elixxir/comms/network/dataStructures" + "gitlab.com/elixxir/primitives/id" + "testing" +) + +//region ERS Memory Map Impl +// Memory map based ExtendedRoundStorage database +type ersMemMap struct { + rounds map[id.Round]*pb.RoundInfo +} + +// Store a new round info object into the map +func (ersm ersMemMap) Store(ri *pb.RoundInfo) error { + rid := id.Round(ri.GetID()) + // See if the round exists, if it does we need to check that the update ID is newer than the current + ori, err := ersm.Retrieve(rid) + if err != nil { + return err + } + + if ori == nil || ori.UpdateID < ri.UpdateID { + ersm.rounds[rid] = ri + } else { + jww.WARN.Printf("Passed in round update ID of %v lower than currently stored ID %v", + ri.UpdateID, ori.UpdateID) + } + + return nil +} + +// Get a round info object from the memory map database +func (ersm ersMemMap) Retrieve(id id.Round) (*pb.RoundInfo, error) { + return ersm.rounds[id], nil +} + +// Get multiple specific round info objects from the memory map database +func (ersm ersMemMap) RetrieveMany(rounds []id.Round) ([]*pb.RoundInfo, error) { + var r []*pb.RoundInfo + + for _, round := range rounds { + ri, err := ersm.Retrieve(round) + if err != nil { + return nil, err + } + + r = append(r, ri) + } + + return r, nil +} + +// Retrieve a concurrent range of round info objects from the memory map database +func (ersm ersMemMap) RetrieveRange(first, last id.Round) ([]*pb.RoundInfo, error) { + idrange := uint64(last - first) + i := uint64(0) + + var r []*pb.RoundInfo + + // for some reason <= doesn't work? + for i < idrange+1 { + ri, err := ersm.Retrieve(id.Round(uint64(first) + i)) + if err != nil { + return nil, err + } + + r = append(r, ri) + i++ + } + + return r, nil +} + +//endregion + +// Test we can insert a round, get it, try to update with an older ID, it doesn't update, and it does update with +// a newer ID +func TestERSStore(t *testing.T) { + // Setup + var ers ds.ExternalRoundStorage = ersMemMap{rounds: make(map[id.Round]*pb.RoundInfo)} + + // Store a test round + r := pb.RoundInfo{ID: 1, UpdateID: 5} + err := ers.Store(&r) + if err != nil { + t.Errorf(err.Error()) + } + + // Test that we get a new round with our info + ri, err := ers.Retrieve(id.Round(r.ID)) + if err != nil { + t.Errorf(err.Error()) + } + if ri == nil { + t.Fatalf("ri object is nil, Retrieve did not return a round") + } + if ri.ID != r.ID && ri.UpdateID != r.UpdateID { + t.Errorf("did not return the same round ID or update ID as we inputted.") + } + + // Update the test round + ru1 := pb.RoundInfo{ID: 1, UpdateID: 3} + err = ers.Store(&ru1) + if err != nil { + t.Errorf(err.Error()) + } + + // Test that this updated round did not get written to the map + riu1, err := ers.Retrieve(id.Round(r.ID)) + if err != nil { + t.Errorf(err.Error()) + } + if riu1 == nil { + t.Fatalf("ri object is nil, Retrieve did not return a round") + } + if riu1.UpdateID == ru1.UpdateID { + t.Errorf("stored round info was updated to have lower update ID of %v", riu1.UpdateID) + } + + // Update the test round + ru2 := pb.RoundInfo{ID: 1, UpdateID: 10} + err = ers.Store(&ru2) + if err != nil { + t.Errorf(err.Error()) + } + + // Test that this updated round did not get written to the map + riu2, err := ers.Retrieve(id.Round(r.ID)) + if err != nil { + t.Errorf(err.Error()) + } + if riu2 == nil { + t.Fatalf("ri object is nil, Retrieve did not return a round") + } + if riu2.UpdateID != ru2.UpdateID { + t.Errorf("stored round info was not updated to have update ID of %v", riu2.UpdateID) + } +} + +// Test that Retrieve has expected behaviour if an item doesn't exist +func TestERSRetrieve(t *testing.T) { + var ers ds.ExternalRoundStorage = ersMemMap{rounds: make(map[id.Round]*pb.RoundInfo)} + ri, err := ers.Retrieve(id.Round(1)) + if err != nil { + t.Errorf(err.Error()) + } + if ri != nil { + t.Errorf("returned round info was not nil") + } + + // Store a test round + r := pb.RoundInfo{ID: 1, UpdateID: 5} + err = ers.Store(&r) + if err != nil { + t.Errorf(err.Error()) + } + + nri, err := ers.Retrieve(id.Round(1)) + if err != nil { + t.Errorf(err.Error()) + } + if nri == nil { + t.Fatalf("returned round info was not nil") + } + if nri.ID != r.ID || nri.UpdateID != r.UpdateID { + t.Errorf("Returned round or update ID did not match what we put in") + } +} + +// Test that the RetrieveMany function will get rounds that are stored, while returning nil with no error for those +// that are not +func TestERSRetrieveMany(t *testing.T) { + // Setup + var ers ds.ExternalRoundStorage = ersMemMap{rounds: make(map[id.Round]*pb.RoundInfo)} + + // Store a test round + origRound1 := pb.RoundInfo{ID: 1, UpdateID: 5} + err := ers.Store(&origRound1) + if err != nil { + t.Errorf(err.Error()) + } + + // Store another test round + origRound2 := pb.RoundInfo{ID: 8, UpdateID: 3} + err = ers.Store(&origRound2) + if err != nil { + t.Errorf(err.Error()) + } + + getRounds := []id.Round{id.Round(origRound1.ID), id.Round(origRound2.ID - 3), id.Round(origRound2.ID)} + returnRounds, err := ers.RetrieveMany(getRounds) + if err != nil { + t.Errorf(err.Error()) + } + + if returnRounds[0] == nil || returnRounds[2] == nil { + t.Fatalf("RetrieveMany did not return a round we expected to get returned") + } + if returnRounds[1] != nil { + t.Errorf("Middle fake round did return a round info object") + } + if returnRounds[0].ID != origRound1.ID || returnRounds[0].UpdateID != origRound1.UpdateID { + t.Errorf("First returned round and original mismatched IDs") + } + if returnRounds[2].ID != origRound2.ID || returnRounds[2].UpdateID != origRound2.UpdateID { + t.Errorf("Second returned round and original mismatched IDs") + } +} + +// Test that the RetrieveRange function will get a range of rounds stored, and return nil with no error for ones that +// are not stored +func TestERSRetrieveRange(t *testing.T) { + // Setup + var ers ds.ExternalRoundStorage = ersMemMap{rounds: make(map[id.Round]*pb.RoundInfo)} + + // Store a test round + origRound1 := pb.RoundInfo{ID: 1, UpdateID: 5} + err := ers.Store(&origRound1) + if err != nil { + t.Errorf(err.Error()) + } + + // Store another test round + origRound2 := pb.RoundInfo{ID: 3, UpdateID: 3} + err = ers.Store(&origRound2) + if err != nil { + t.Errorf(err.Error()) + } + + returnRounds, err := ers.RetrieveRange(1, 3) + if err != nil { + t.Errorf(err.Error()) + } + + if returnRounds[0] == nil || returnRounds[2] == nil { + t.Fatalf("RetrieveMany did not return a round we expected to get returned") + } + if returnRounds[1] != nil { + t.Errorf("Middle fake round did return a round info object") + } + if returnRounds[0].ID != origRound1.ID || returnRounds[0].UpdateID != origRound1.UpdateID { + t.Errorf("First returned round and original mismatched IDs") + } + if returnRounds[2].ID != origRound2.ID || returnRounds[2].UpdateID != origRound2.UpdateID { + t.Errorf("Second returned round and original mismatched IDs") + } +} diff --git a/network/instance.go b/network/instance.go index 5f0f2958b3d677a9809ecf6281157cab8ac6e91b..35e80b8b3b54b35fcef3c4bc95bead129bf9493b 100644 --- a/network/instance.go +++ b/network/instance.go @@ -32,12 +32,14 @@ type Instance struct { full *SecuredNdf roundUpdates *ds.Updates roundData *ds.Data + ers ds.ExternalRoundStorage ipOverride *ds.IpOverrideList } // Initializer for instance structs from base comms and NDF -func NewInstance(c *connect.ProtoComms, partial, full *ndf.NetworkDefinition) (*Instance, error) { +func NewInstance(c *connect.ProtoComms, partial, full *ndf.NetworkDefinition, + ers ds.ExternalRoundStorage) (*Instance, error) { var partialNdf *SecuredNdf var fullNdf *SecuredNdf var err error @@ -100,6 +102,10 @@ func NewInstance(c *connect.ProtoComms, partial, full *ndf.NetworkDefinition) (* } } + if ers != nil { + i.ers = ers + } + return i, nil } @@ -109,7 +115,7 @@ func NewInstanceTesting(c *connect.ProtoComms, partial, full *ndf.NetworkDefinit if t == nil { panic("This is a utility function for testing purposes only!") } - instance, err := NewInstance(c, partial, full) + instance, err := NewInstance(c, partial, full, nil) if err != nil { return nil, errors.Errorf("Unable to create instance: %+v", err) } @@ -283,6 +289,10 @@ func (i *Instance) RoundUpdate(info *pb.RoundInfo) error { return err } + if i.ers != nil { + err = i.ers.Store(info) + } + return nil } diff --git a/network/instance_test.go b/network/instance_test.go index 660f4fb72866d2c3612a955186ff1fc0862f5607..f7869a917894669514c3e3799d243e8fe0a47b60 100644 --- a/network/instance_test.go +++ b/network/instance_test.go @@ -117,7 +117,7 @@ func TestNewInstanceTesting_Error(t *testing.T) { //tests newInstance errors properly when there is no NDF func TestNewInstance_NilNDFs(t *testing.T) { - _, err := NewInstance(&connect.ProtoComms{}, nil, nil) + _, err := NewInstance(&connect.ProtoComms{}, nil, nil, nil) if err == nil { t.Errorf("Creation of NewInstance without an ndf succeded") } else if !strings.Contains(err.Error(), "Cannot create a network "+ @@ -200,7 +200,7 @@ func setupComm(t *testing.T) (*Instance, *mixmessages.NDF) { err = signature.Sign(f, privKey) pc := &connect.ProtoComms{} - i, err := NewInstance(pc, baseNDF, baseNDF) + i, err := NewInstance(pc, baseNDF, baseNDF, nil) if err != nil { t.Error(nil) } @@ -223,7 +223,7 @@ func TestInstance_RoundUpdate(t *testing.T) { privKey, err := rsa.LoadPrivateKeyFromPem(priv) err = signature.Sign(msg, privKey) - i, err := NewInstance(&connect.ProtoComms{}, testutils.NDF, testutils.NDF) + i, err := NewInstance(&connect.ProtoComms{}, testutils.NDF, testutils.NDF, nil) pub := testkeys.LoadFromPath(testkeys.GetGatewayCertPath()) err = i.RoundUpdate(msg) if err == nil {