diff --git a/api/authenticatedChannel.go b/api/authenticatedChannel.go index 47228cff42516de89903c7090d6fd33eb8e74054..7e995a896a69e1d045186aa377b9ada99d5c4087 100644 --- a/api/authenticatedChannel.go +++ b/api/authenticatedChannel.go @@ -9,6 +9,7 @@ package api import ( "encoding/binary" + "gitlab.com/elixxir/client/catalog" "math/rand" "github.com/cloudflare/circl/dh/sidh" @@ -16,7 +17,6 @@ import ( jww "github.com/spf13/jwalterweatherman" "gitlab.com/elixxir/client/auth" "gitlab.com/elixxir/client/interfaces" - "gitlab.com/elixxir/client/interfaces/preimage" "gitlab.com/elixxir/client/storage/edge" util "gitlab.com/elixxir/client/storage/utility" "gitlab.com/elixxir/crypto/contact" @@ -154,28 +154,28 @@ func (c *Client) MakePrecannedAuthenticatedChannel(precannedID uint) (contact.Co c.storage.GetEdge().Add(edge.Preimage{ Data: sessionPartner.GetE2EPreimage(), - Type: preimage.E2e, + Type: catalog.E2e, Source: precan.ID[:], }, me) // slient (rekey) c.storage.GetEdge().Add(edge.Preimage{ Data: sessionPartner.GetSilentPreimage(), - Type: preimage.Silent, + Type: catalog.Silent, Source: precan.ID[:], }, me) // File transfer end c.storage.GetEdge().Add(edge.Preimage{ Data: sessionPartner.GetFileTransferPreimage(), - Type: preimage.EndFT, + Type: catalog.EndFT, Source: precan.ID[:], }, me) // group request c.storage.GetEdge().Add(edge.Preimage{ Data: sessionPartner.GetGroupRequestPreimage(), - Type: preimage.GroupRq, + Type: catalog.GroupRq, Source: precan.ID[:], }, me) diff --git a/api/client.go b/api/client.go index 9b640a694737924f0897b31231b2c33fe7df4ff3..17ea0a289479a309311e9b8e721b91bedf87733d 100644 --- a/api/client.go +++ b/api/client.go @@ -12,6 +12,7 @@ import ( "github.com/pkg/errors" jww "github.com/spf13/jwalterweatherman" "gitlab.com/elixxir/client/auth" + "gitlab.com/elixxir/client/catalog" "gitlab.com/elixxir/client/event" "gitlab.com/elixxir/client/interfaces" "gitlab.com/elixxir/client/interfaces/params" @@ -795,7 +796,7 @@ func (c *Client) DeleteContact(partnerId *id.ID) error { //delete the preimages if err = c.storage.GetEdge().Remove(edge.Preimage{ Data: e2ePreimage, - Type: preimage.E2e, + Type: catalog.E2e, Source: partnerId[:], }, c.storage.GetUser().ReceptionID); err != nil { jww.WARN.Printf("Failed delete the preimage for e2e "+ @@ -804,7 +805,7 @@ func (c *Client) DeleteContact(partnerId *id.ID) error { if err = c.storage.GetEdge().Remove(edge.Preimage{ Data: rekeyPreimage, - Type: preimage.Silent, + Type: catalog.Silent, Source: partnerId[:], }, c.storage.GetUser().ReceptionID); err != nil { jww.WARN.Printf("Failed delete the preimage for rekey "+ @@ -813,7 +814,7 @@ func (c *Client) DeleteContact(partnerId *id.ID) error { if err = c.storage.GetEdge().Remove(edge.Preimage{ Data: fileTransferPreimage, - Type: preimage.EndFT, + Type: catalog.EndFT, Source: partnerId[:], }, c.storage.GetUser().ReceptionID); err != nil { jww.WARN.Printf("Failed delete the preimage for file transfer "+ @@ -822,7 +823,7 @@ func (c *Client) DeleteContact(partnerId *id.ID) error { if err = c.storage.GetEdge().Remove(edge.Preimage{ Data: groupRequestPreimage, - Type: preimage.GroupRq, + Type: catalog.GroupRq, Source: partnerId[:], }, c.storage.GetUser().ReceptionID); err != nil { jww.WARN.Printf("Failed delete the preimage for group request "+ @@ -974,7 +975,7 @@ func checkVersionAndSetupStorage(def *ndf.NetworkDefinition, //add the request preiamge storageSess.GetEdge().Add(edge.Preimage{ Data: preimage.GenerateRequest(protoUser.ReceptionID), - Type: preimage.Request, + Type: catalog.Request, Source: protoUser.ReceptionID[:], }, protoUser.ReceptionID) diff --git a/auth/callback.go b/auth/callback.go index 70b9c2843406205198cdb439666741b193c52518..8b56de709682876fcf051e760a3c52baa7c9d53a 100644 --- a/auth/callback.go +++ b/auth/callback.go @@ -9,6 +9,7 @@ package auth import ( "fmt" + "gitlab.com/elixxir/client/catalog" "strings" "github.com/cloudflare/circl/dh/sidh" @@ -490,8 +491,8 @@ func (m *Manager) doConfirm(sr *auth.SentRequest, grp *cyclic.Group, //remove the confirm fingerprint fp := sr.GetFingerprint() if err := m.storage.GetEdge().Remove(edge.Preimage{ - Data: preimage.Generate(fp[:], preimage.Confirm), - Type: preimage.Confirm, + Data: preimage.Generate(fp[:], catalog.Confirm), + Type: catalog.Confirm, Source: sr.GetPartner()[:], }, m.storage.GetUser().ReceptionID); err != nil { jww.WARN.Printf("Failed delete the preimage for confirm from %s: %+v", diff --git a/auth/confirm.go b/auth/confirm.go index 28a5292aec2ad83f888c35e1f19feb32a55acea7..59e374b9d589275065118ab6ece63b86aad5614e 100644 --- a/auth/confirm.go +++ b/auth/confirm.go @@ -9,6 +9,7 @@ package auth import ( "fmt" + "gitlab.com/elixxir/client/catalog" "github.com/pkg/errors" jww "github.com/spf13/jwalterweatherman" @@ -98,7 +99,7 @@ func (m *Manager) ConfirmRequestAuth(partner contact.Contact) (id.Round, error) // get the fingerprint from the old ownership proof fp := cAuth.MakeOwnershipProofFP(storedContact.OwnershipProof) - preimg := preimage.Generate(fp[:], preimage.Confirm) + preimg := preimage.Generate(fp[:], catalog.Confirm) // final construction baseFmt.SetEcrPayload(ecrPayload) @@ -186,10 +187,10 @@ func addPreimages(partner *id.ID, store *storage.Session) { for i := range existingEdges { delete := true switch existingEdges[i].Type { - case preimage.E2e: - case preimage.Silent: - case preimage.EndFT: - case preimage.GroupRq: + case catalog.E2e: + case catalog.Silent: + case catalog.EndFT: + case catalog.GroupRq: default: delete = false } @@ -209,28 +210,28 @@ func addPreimages(partner *id.ID, store *storage.Session) { // e2e store.GetEdge().Add(edge.Preimage{ Data: sessionPartner.GetE2EPreimage(), - Type: preimage.E2e, + Type: catalog.E2e, Source: partner[:], }, me) // silent (rekey) store.GetEdge().Add(edge.Preimage{ Data: sessionPartner.GetSilentPreimage(), - Type: preimage.Silent, + Type: catalog.Silent, Source: partner[:], }, me) // File transfer end store.GetEdge().Add(edge.Preimage{ Data: sessionPartner.GetFileTransferPreimage(), - Type: preimage.EndFT, + Type: catalog.EndFT, Source: partner[:], }, me) // group Request store.GetEdge().Add(edge.Preimage{ Data: sessionPartner.GetGroupRequestPreimage(), - Type: preimage.GroupRq, + Type: catalog.GroupRq, Source: partner[:], }, me) } diff --git a/auth/request.go b/auth/request.go index 91cda7a3f324bf7f14ae6b8052e3caccf725767c..650d4f4ac314a9fe5cc31e0160f22becfad00c81 100644 --- a/auth/request.go +++ b/auth/request.go @@ -8,6 +8,7 @@ package auth import ( + "gitlab.com/elixxir/client/catalog" e2e2 "gitlab.com/elixxir/client/e2e/ratchet" "io" "strings" @@ -156,8 +157,8 @@ func requestAuth(partner, me contact.Contact, rng io.Reader, reset bool, contents := request.Marshal() storage.GetEdge().Add(edge.Preimage{ - Data: preimage.Generate(confirmFp[:], preimage.Confirm), - Type: preimage.Confirm, + Data: preimage.Generate(confirmFp[:], catalog.Confirm), + Type: catalog.Confirm, Source: partner.ID[:], }, me.ID) diff --git a/catalog/messageTypes.go b/catalog/messageTypes.go new file mode 100644 index 0000000000000000000000000000000000000000..a5fc5c9b442417b7e6d642267bf3ed70ca53b06c --- /dev/null +++ b/catalog/messageTypes.go @@ -0,0 +1,36 @@ +package catalog + +type MessageType uint32 + +const MessageTypeLen = 32 / 8 + +const ( + /*general message types*/ + + // NoType - Used as a wildcard for listeners to listen to all existing types. + // Think of it as "No type in particular" + NoType MessageType = 0 + + // XxMessage - Type of message sent by the xx messenger. + XxMessage = 2 + + /*End to End Rekey message types*/ + + // KeyExchangeTrigger - Trigger a rekey, this message is used locally in client only + KeyExchangeTrigger = 30 + // KeyExchangeConfirm - Rekey confirmation message. Sent by partner to confirm completion of a rekey + KeyExchangeConfirm = 31 + + /* Group chat message types */ + + // GroupCreationRequest - A group chat request message sent to all members in a group. + GroupCreationRequest = 40 + + // NewFileTransfer is transmitted first on the initialization of a file + // transfer to inform the receiver about the incoming file. + NewFileTransfer = 50 + + // EndFileTransfer is sent once all file parts have been transmitted to + // inform the receiver that the file transfer has ended. + EndFileTransfer = 51 +) diff --git a/interfaces/preimage/types.go b/catalog/services.go similarity index 65% rename from interfaces/preimage/types.go rename to catalog/services.go index bd6226e623594561ef0338210af8824c1b789f27..9c32b32522508dbacc1ddaf7f14bc1708a1c1f68 100644 --- a/interfaces/preimage/types.go +++ b/catalog/services.go @@ -1,7 +1,9 @@ -package preimage +package catalog + +import "gitlab.com/elixxir/crypto/sih" const ( - Default = "default" + Default = sih.Default Request = "request" Confirm = "confirm" Silent = "silent" diff --git a/e2e/parse/firstMessagePart.go b/e2e/parse/firstMessagePart.go new file mode 100644 index 0000000000000000000000000000000000000000..fa67b64411e0f2272178452eeb0a8deb91612271 --- /dev/null +++ b/e2e/parse/firstMessagePart.go @@ -0,0 +1,131 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright © 2020 xx network SEZC // +// // +// Use of this source code is governed by a license that can be found in the // +// LICENSE file // +/////////////////////////////////////////////////////////////////////////////// + +package parse + +import ( + "encoding/binary" + "gitlab.com/elixxir/client/interfaces/message" + "time" +) + +// Sizes of message parts, in bytes. +const ( + numPartsLen = 1 + typeLen = message.TypeLen + timestampLen = 8 + firstPartVerLen = 1 + firstHeaderLen = headerLen + numPartsLen + typeLen + timestampLen + firstPartVerLen +) + +// The current version of the firstMessagePart message format. +const firstMessagePartCurrentVersion = 0 + +type firstMessagePart struct { + messagePart + NumParts []byte + Type []byte + Timestamp []byte + Version []byte // Version of the message format; always the last bit +} + +// newFirstMessagePart creates a new firstMessagePart for the passed in +// contents. Does no length checks. +func newFirstMessagePart(mt message.Type, id uint32, numParts uint8, + timestamp time.Time, contents []byte) firstMessagePart { + + // Create the message structure + m := FirstMessagePartFromBytes(make([]byte, len(contents)+firstHeaderLen)) + + // Set the message type + binary.BigEndian.PutUint32(m.Type, uint32(mt)) + + // Set the message ID + binary.BigEndian.PutUint32(m.Id, id) + + // Set the part number. It is always zero because this is the first part. + // Because the default is zero this step could be skipped, but keep it in + // the code for clarity. + m.Part[0] = 0 + + // Set the number of parts to the message + m.NumParts[0] = numParts + + // Set the timestamp as unix nano + binary.BigEndian.PutUint64(m.Timestamp, uint64(timestamp.UnixNano())) + + // Set the length of the contents + binary.BigEndian.PutUint16(m.Len, uint16(len(contents))) + + // Set the contents + copy(m.Contents[:len(contents)], contents) + + // Set the version number + m.Version[0] = firstMessagePartCurrentVersion + + return m +} + +// Map of firstMessagePart encoding version numbers to their map functions. +var firstMessagePartFromBytesVersions = map[uint8]func([]byte) firstMessagePart{ + firstMessagePartCurrentVersion: firstMessagePartFromBytesVer0, +} + +// FirstMessagePartFromBytes builds a firstMessagePart mapped to the passed in +// data slice. Mapped by reference; a copy is not made. +func FirstMessagePartFromBytes(data []byte) firstMessagePart { + + // Map the data according to its version + version := data[len(data)-1] + mapFunc, exists := firstMessagePartFromBytesVersions[version] + if exists { + return mapFunc(data) + } + + return firstMessagePart{} +} + +func firstMessagePartFromBytesVer0(data []byte) firstMessagePart { + return firstMessagePart{ + messagePart: messagePart{ + Data: data, + Id: data[:idLen], + Part: data[idLen : idLen+partLen], + Len: data[idLen+partLen : idLen+partLen+lenLen], + Contents: data[idLen+partLen+lenLen+numPartsLen+typeLen+timestampLen : len(data)-firstPartVerLen-1], + }, + NumParts: data[idLen+partLen+lenLen : idLen+partLen+lenLen+numPartsLen], + Type: data[idLen+partLen+lenLen+numPartsLen : idLen+partLen+lenLen+numPartsLen+typeLen], + Timestamp: data[idLen+partLen+lenLen+numPartsLen+typeLen : idLen+partLen+lenLen+numPartsLen+typeLen+timestampLen], + Version: data[len(data)-firstPartVerLen:], + } +} + +// GetType returns the message type. +func (m firstMessagePart) GetType() message.Type { + return message.Type(binary.BigEndian.Uint32(m.Type)) +} + +// GetNumParts returns the number of message parts. +func (m firstMessagePart) GetNumParts() uint8 { + return m.NumParts[0] +} + +// GetTimestamp returns the timestamp as a time.Time. +func (m firstMessagePart) GetTimestamp() time.Time { + return time.Unix(0, int64(binary.BigEndian.Uint64(m.Timestamp))) +} + +// GetVersion returns the version number of the data encoding. +func (m firstMessagePart) GetVersion() uint8 { + return m.Version[0] +} + +// Bytes returns the serialised message data. +func (m firstMessagePart) Bytes() []byte { + return m.Data +} diff --git a/e2e/parse/firstMessagePart_test.go b/e2e/parse/firstMessagePart_test.go new file mode 100644 index 0000000000000000000000000000000000000000..9902790552e45add0d651b5db38e3e277173b39e --- /dev/null +++ b/e2e/parse/firstMessagePart_test.go @@ -0,0 +1,95 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright © 2020 xx network SEZC // +// // +// Use of this source code is governed by a license that can be found in the // +// LICENSE file // +/////////////////////////////////////////////////////////////////////////////// + +package parse + +import ( + "bytes" + "gitlab.com/elixxir/client/interfaces/message" + "reflect" + "testing" + "time" +) + +// Expected firstMessagePart for checking against, generated by fmp in TestNewFirstMessagePart +var efmp = firstMessagePart{ + messagePart: messagePart{ + Data: []byte{0, 0, 4, 53, 0, 0, 13, 2, 0, 0, 0, 2, 22, 87, 28, 11, 215, + 220, 82, 0, 116, 101, 115, 116, 105, 110, 103, 115, 116, 114, 105, + 110, 103, 0, firstMessagePartCurrentVersion}, + Id: []byte{0, 0, 4, 53}, + Part: []byte{0}, + Len: []byte{0, 13}, + Contents: []byte{116, 101, 115, 116, 105, 110, 103, 115, 116, 114, 105, + 110, 103}, + }, + NumParts: []byte{2}, + Type: []byte{0, 0, 0, 2}, + Timestamp: []byte{22, 87, 28, 11, 215, 220, 82, 0}, + Version: []byte{firstMessagePartCurrentVersion}, +} + +// Test that newFirstMessagePart returns a correctly made firstMessagePart +func TestNewFirstMessagePart(t *testing.T) { + fmp := newFirstMessagePart( + message.XxMessage, + 1077, + 2, + time.Unix(1609786229, 0).UTC(), + []byte{'t', 'e', 's', 't', 'i', 'n', 'g', 's', 't', 'r', 'i', 'n', 'g'}, + ) + + gotTime := fmp.GetTimestamp() + expectedTime := time.Unix(1609786229, 0).UTC() + if !gotTime.Equal(expectedTime) { + t.Errorf("Failed to get expected timestamp."+ + "\nexpected: %s\nreceived: %s", expectedTime, gotTime) + } + + if !reflect.DeepEqual(fmp, efmp) { + t.Errorf("Expected and got firstMessagePart did not match."+ + "\nexpected: %+v\nrecieved: %+v", efmp, fmp) + } +} + +// Test that FirstMessagePartFromBytes returns a correctly made firstMessagePart from the bytes of one +func TestFirstMessagePartFromBytes(t *testing.T) { + fmp := FirstMessagePartFromBytes(efmp.Data) + + if !reflect.DeepEqual(fmp, efmp) { + t.Error("Expected and got firstMessagePart did not match") + } +} + +// Test that GetType returns the correct type for a firstMessagePart +func TestFirstMessagePart_GetType(t *testing.T) { + if efmp.GetType() != message.XxMessage { + t.Errorf("Got %v, expected %v", efmp.GetType(), message.XxMessage) + } +} + +// Test that GetNumParts returns the correct number of parts for a firstMessagePart +func TestFirstMessagePart_GetNumParts(t *testing.T) { + if efmp.GetNumParts() != 2 { + t.Errorf("Got %v, expected %v", efmp.GetNumParts(), 2) + } +} + +// Test that GetTimestamp returns the correct timestamp for a firstMessagePart +func TestFirstMessagePart_GetTimestamp(t *testing.T) { + et := efmp.GetTimestamp() + if !time.Unix(1609786229, 0).Equal(et) { + t.Errorf("Got %v, expected %v", et, time.Unix(1609786229, 0)) + } +} + +// Test that GetTimestamp returns the correct bytes for a firstMessagePart +func TestFirstMessagePart_Bytes(t *testing.T) { + if bytes.Compare(efmp.Bytes(), efmp.Data) != 0 { + t.Errorf("Got %v, expected %v", efmp.Bytes(), efmp.Data) + } +} diff --git a/e2e/parse/messagePart.go b/e2e/parse/messagePart.go new file mode 100644 index 0000000000000000000000000000000000000000..01c514d7c33fe1654534a67aeaf8671744d93982 --- /dev/null +++ b/e2e/parse/messagePart.go @@ -0,0 +1,118 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright © 2020 xx network SEZC // +// // +// Use of this source code is governed by a license that can be found in the // +// LICENSE file // +/////////////////////////////////////////////////////////////////////////////// + +package parse + +import ( + "encoding/binary" +) + +// Sizes of message parts, in bytes. +const ( + idLen = 4 + partLen = 1 + lenLen = 2 + partVerLen = 1 + headerLen = idLen + partLen + lenLen + partVerLen +) + +// The current version of the messagePart message format. +const messagePartCurrentVersion = 0 + +type messagePart struct { + Data []byte + Id []byte + Part []byte + Len []byte + Contents []byte + Version []byte // Version of the message format; always the last bit +} + +// newMessagePart creates a new messagePart for the passed in contents. Does no +// length checks. +func newMessagePart(id uint32, part uint8, contents []byte) messagePart { + // Create the message structure + data := make([]byte, len(contents)+headerLen) + m := messagePartFromBytes(data) + + // Set the message ID + binary.BigEndian.PutUint32(m.Id, id) + + // Set the message part number + m.Part[0] = part + + // Set the contents length + binary.BigEndian.PutUint16(m.Len, uint16(len(contents))) + + // Copy the contents into the message + copy(m.Contents[:len(contents)], contents) + + // Set the version number + m.Version[0] = messagePartCurrentVersion + + return m +} + +// Map of messagePart encoding version numbers to their map functions. +var messagePartFromBytesVersions = map[uint8]func([]byte) messagePart{ + messagePartCurrentVersion: messagePartFromBytesVer0, +} + +// messagePartFromBytes builds a messagePart mapped to the passed in data slice. +// Mapped by reference; a copy is not made. +func messagePartFromBytes(data []byte) messagePart { + + // Map the data according to its version + version := data[len(data)-1] + mapFunc, exists := messagePartFromBytesVersions[version] + if exists { + return mapFunc(data) + } + + return messagePart{} +} + +func messagePartFromBytesVer0(data []byte) messagePart { + return messagePart{ + Data: data, + Id: data[:idLen], + Part: data[idLen : idLen+partLen], + Len: data[idLen+partLen : idLen+partLen+lenLen], + Contents: data[idLen+partLen+lenLen : len(data)-partVerLen], + Version: data[len(data)-partVerLen:], + } +} + +// GetID returns the message ID. +func (m messagePart) GetID() uint32 { + return binary.BigEndian.Uint32(m.Id) +} + +// GetPart returns the message part number. +func (m messagePart) GetPart() uint8 { + return m.Part[0] +} + +// GetContents returns the entire contents slice. +func (m messagePart) GetContents() []byte { + return m.Contents +} + +// GetSizedContents returns the contents truncated to include only stored data. +func (m messagePart) GetSizedContents() []byte { + return m.Contents[:m.GetContentsLength()] +} + +// GetContentsLength returns the length of the data in the contents. +func (m messagePart) GetContentsLength() int { + return int(binary.BigEndian.Uint16(m.Len)) +} + +// Bytes returns the serialised message data. +func (m messagePart) Bytes() []byte { + return m.Data +} diff --git a/e2e/parse/messagePart_test.go b/e2e/parse/messagePart_test.go new file mode 100644 index 0000000000000000000000000000000000000000..61ecc7467ff5c95dd2cf89830cacbb319cf9660f --- /dev/null +++ b/e2e/parse/messagePart_test.go @@ -0,0 +1,72 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright © 2020 xx network SEZC // +// // +// Use of this source code is governed by a license that can be found in the // +// LICENSE file // +/////////////////////////////////////////////////////////////////////////////// + +package parse + +import ( + "bytes" + "reflect" + "testing" +) + +// Expected messagePart for checking against, generated by gotmp in Test_newMessagePart +var emp = messagePart{ + Data: []uint8{0x0, 0x0, 0x0, 0x20, 0x6, 0x0, 0x7, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, messagePartCurrentVersion}, + Id: []uint8{0x0, 0x0, 0x0, 0x20}, Part: []uint8{0x6}, + Len: []uint8{0x0, 0x7}, + Contents: []uint8{0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67}, + Version: []uint8{messagePartCurrentVersion}, +} + +// This tests that a new function part is successfully created +func Test_newMessagePart(t *testing.T) { + gotmp := newMessagePart(32, 6, []byte{'t', 'e', 's', 't', 'i', 'n', 'g'}) + if !reflect.DeepEqual(gotmp, emp) { + t.Errorf("MessagePart received and MessagePart expected do not match."+ + "\nexpected: %#v\nreceived: %#v", emp, gotmp) + } +} + +// Test that GetID returns the correct ID +func TestMessagePart_GetID(t *testing.T) { + if emp.GetID() != 32 { + t.Errorf("received and expected do not match."+ + "\n\tGot: %#v\n\tExpected: %#v", emp.GetID(), 32) + } +} + +// Test that GetPart returns the correct part number +func TestMessagePart_GetPart(t *testing.T) { + if emp.GetPart() != 6 { + t.Errorf("received and expected do not match."+ + "\n\tGot: %#v\n\tExpected: %#v", emp.GetPart(), 6) + } +} + +// Test that GetContents returns the message contests +func TestMessagePart_GetContents(t *testing.T) { + if bytes.Compare(emp.GetContents(), []byte{'t', 'e', 's', 't', 'i', 'n', 'g'}) != 0 { + t.Errorf("received and expected do not match."+ + "\n\tGot: %#v\n\tExpected: %#v", emp.GetContents(), 6) + } +} + +// Test that GetSizedContents returns the message contests +func TestMessagePart_GetSizedContents(t *testing.T) { + if bytes.Compare(emp.GetSizedContents(), []byte{'t', 'e', 's', 't', 'i', 'n', 'g'}) != 0 { + t.Errorf("received and expected do not match."+ + "\n\tGot: %#v\n\tExpected: %#v", emp.GetSizedContents(), 6) + } +} + +// Test that GetContentsLength returns the message length +func TestMessagePart_GetContentsLength(t *testing.T) { + if emp.GetContentsLength() != 7 { + t.Errorf("received and expected do not match."+ + "\n\tGot: %#v\n\tExpected: %#v", emp.GetContentsLength(), 7) + } +} diff --git a/e2e/parse/partition.go b/e2e/parse/partition.go new file mode 100644 index 0000000000000000000000000000000000000000..ad66ba2f30e09434dc090f07e0917137973d9b00 --- /dev/null +++ b/e2e/parse/partition.go @@ -0,0 +1,108 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright © 2020 xx network SEZC // +// // +// Use of this source code is governed by a license that can be found in the // +// LICENSE file // +/////////////////////////////////////////////////////////////////////////////// + +package parse + +import ( + "github.com/pkg/errors" + "gitlab.com/elixxir/client/interfaces/message" + "gitlab.com/elixxir/client/storage" + "gitlab.com/xx_network/primitives/id" + "gitlab.com/xx_network/primitives/netTime" + "time" +) + +const MaxMessageParts = 255 + +type Partitioner struct { + baseMessageSize int + firstContentsSize int + partContentsSize int + deltaFirstPart int + maxSize int + session *storage.Session +} + +func NewPartitioner(messageSize int, session *storage.Session) Partitioner { + p := Partitioner{ + baseMessageSize: messageSize, + firstContentsSize: messageSize - firstHeaderLen, + partContentsSize: messageSize - headerLen, + deltaFirstPart: firstHeaderLen - headerLen, + session: session, + } + p.maxSize = p.firstContentsSize + (MaxMessageParts-1)*p.partContentsSize + return p +} + +func (p Partitioner) Partition(recipient *id.ID, mt message.Type, + timestamp time.Time, payload []byte) ([][]byte, uint64, error) { + + if len(payload) > p.maxSize { + return nil, 0, errors.Errorf("Payload is too long, max payload "+ + "length is %d, received %d", p.maxSize, len(payload)) + } + + // Get the ID of the sent message + fullMessageID, messageID := p.session.Conversations().Get(recipient).GetNextSendID() + + // Get the number of parts of the message; this equates to just a linear + // equation + numParts := uint8((len(payload) + p.deltaFirstPart + p.partContentsSize - 1) / p.partContentsSize) + parts := make([][]byte, numParts) + + // Create the first message part + var sub []byte + sub, payload = splitPayload(payload, p.firstContentsSize) + parts[0] = newFirstMessagePart(mt, messageID, numParts, timestamp, sub).Bytes() + + // Create all subsequent message parts + for i := uint8(1); i < numParts; i++ { + sub, payload = splitPayload(payload, p.partContentsSize) + parts[i] = newMessagePart(messageID, i, sub).Bytes() + } + + return parts, fullMessageID, nil +} + +func (p Partitioner) HandlePartition(sender *id.ID, _ message.EncryptionType, + contents []byte, relationshipFingerprint []byte) (message.Receive, bool) { + + if isFirst(contents) { + // If it is the first message in a set, then handle it as so + + // Decode the message structure + fm := FirstMessagePartFromBytes(contents) + + // Handle the message ID + messageID := p.session.Conversations().Get(sender). + ProcessReceivedMessageID(fm.GetID()) + storeageTimestamp := netTime.Now() + return p.session.Partition().AddFirst(sender, fm.GetType(), + messageID, fm.GetPart(), fm.GetNumParts(), fm.GetTimestamp(), storeageTimestamp, + fm.GetSizedContents(), relationshipFingerprint) + } else { + // If it is a subsequent message part, handle it as so + mp := messagePartFromBytes(contents) + messageID := p.session.Conversations().Get(sender). + ProcessReceivedMessageID(mp.GetID()) + + return p.session.Partition().Add(sender, messageID, mp.GetPart(), + mp.GetSizedContents(), relationshipFingerprint) + } +} + +func splitPayload(payload []byte, length int) ([]byte, []byte) { + if len(payload) < length { + return payload, payload + } + return payload[:length], payload[length:] +} + +func isFirst(payload []byte) bool { + return payload[idLen] == 0 +} diff --git a/e2e/parse/partition_test.go b/e2e/parse/partition_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7c063f2d222d9171416a7d0366325dd464da7475 --- /dev/null +++ b/e2e/parse/partition_test.go @@ -0,0 +1,106 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright © 2020 xx network SEZC // +// // +// Use of this source code is governed by a license that can be found in the // +// LICENSE file // +/////////////////////////////////////////////////////////////////////////////// + +package parse + +import ( + "gitlab.com/elixxir/client/interfaces/message" + "gitlab.com/elixxir/client/storage" + "gitlab.com/xx_network/primitives/id" + "gitlab.com/xx_network/primitives/netTime" + "testing" +) + +var ipsumTestStr = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras sit amet euismod est. Donec dolor " + + "neque, efficitur et interdum eu, lacinia quis mi. Duis bibendum elit ac lacus finibus pharetra. Suspendisse " + + "blandit erat in odio faucibus consectetur. Suspendisse sed consequat purus. Curabitur fringilla mi sit amet odio " + + "interdum suscipit. Etiam vitae dui posuere, congue mi a, convallis odio. In commodo risus at lorem volutpat " + + "placerat. In cursus magna purus, suscipit dictum lorem aliquam non. Praesent efficitur." + +// Test that NewPartitioner outputs a correctly made Partitioner +func TestNewPartitioner(t *testing.T) { + storeSession := storage.InitTestingSession(t) + p := NewPartitioner(4096, storeSession) + + if p.baseMessageSize != 4096 { + t.Errorf("baseMessageSize content mismatch"+ + "\n\texpected: %v\n\treceived: %v", + 4096, p.baseMessageSize) + } + + if p.deltaFirstPart != firstHeaderLen-headerLen { + t.Errorf("deltaFirstPart content mismatch"+ + "\n\texpected: %v\n\treceived: %v", + firstHeaderLen-headerLen, p.deltaFirstPart) + } + + if p.firstContentsSize != 4096-firstHeaderLen { + t.Errorf("firstContentsSize content mismatch"+ + "\n\texpected: %v\n\treceived: %v", + 4096-firstHeaderLen, p.firstContentsSize) + } + + if p.maxSize != (4096-firstHeaderLen)+(MaxMessageParts-1)*(4096-headerLen) { + t.Errorf("maxSize content mismatch"+ + "\n\texpected: %v\n\treceived: %v", + (4096-firstHeaderLen)+(MaxMessageParts-1)*(4096-headerLen), p.maxSize) + } + + if p.partContentsSize != 4088 { + t.Errorf("partContentsSize content mismatch"+ + "\n\texpected: %v\n\treceived: %v", + 4088, p.partContentsSize) + } + + if p.session != storeSession { + t.Errorf("session content mismatch") + } +} + +// Test that no error is returned running Partition +func TestPartitioner_Partition(t *testing.T) { + storeSession := storage.InitTestingSession(t) + p := NewPartitioner(len(ipsumTestStr), storeSession) + + _, _, err := p.Partition(&id.DummyUser, message.XxMessage, + netTime.Now(), []byte(ipsumTestStr)) + if err != nil { + t.Error(err) + } +} + +// Test that HandlePartition can handle a message part +func TestPartitioner_HandlePartition(t *testing.T) { + storeSession := storage.InitTestingSession(t) + p := NewPartitioner(len(ipsumTestStr), storeSession) + + m := newMessagePart(1107, 1, []byte(ipsumTestStr)) + + _, _ = p.HandlePartition( + &id.DummyUser, + message.None, + m.Bytes(), + []byte{'t', 'e', 's', 't', 'i', 'n', 'g', + 's', 't', 'r', 'i', 'n', 'g'}, + ) +} + +// Test that HandlePartition can handle a first message part +func TestPartitioner_HandleFirstPartition(t *testing.T) { + storeSession := storage.InitTestingSession(t) + p := NewPartitioner(len(ipsumTestStr), storeSession) + + m := newFirstMessagePart(message.XxMessage, 1107, 1, netTime.Now(), []byte(ipsumTestStr)) + + _, _ = p.HandlePartition( + &id.DummyUser, + message.None, + m.Bytes(), + []byte{'t', 'e', 's', 't', 'i', 'n', 'g', + 's', 't', 'r', 'i', 'n', 'g'}, + ) +} diff --git a/e2e/ratchet/interface.go b/e2e/ratchet/interface.go deleted file mode 100644 index 4e1bb580bf4266eb7791426043d75ec48356ae3f..0000000000000000000000000000000000000000 --- a/e2e/ratchet/interface.go +++ /dev/null @@ -1,27 +0,0 @@ -package ratchet - -import ( - "github.com/cloudflare/circl/dh/sidh" - "gitlab.com/elixxir/client/e2e/ratchet/partner" - "gitlab.com/elixxir/client/e2e/ratchet/partner/session" - "gitlab.com/elixxir/client/network/message" - "gitlab.com/elixxir/crypto/cyclic" - "gitlab.com/xx_network/primitives/id" -) - -type Ratchet2 interface { - // AddPartner adds a partner. Automatically creates both send and receive - // sessions using the passed cryptographic data and per the parameters sent - // - AddPartner(partnerID *id.ID, partnerPubKey, myPrivKey *cyclic.Int, - partnerSIDHPubKey *sidh.PublicKey, mySIDHPrivKey *sidh.PrivateKey, - sendParams, receiveParams session.Params) - - GetPartner(partnerID *id.ID) (*partner.Manager, error) - DeletePartner(partnerId *id.ID) - GetAllPartnerIDs() []*id.ID - GetDHPrivateKey() *cyclic.Int - GetDHPublicKey() *cyclic.Int - AddService(tag string, processor message.Processor) - RemoveService(tag string) -} diff --git a/switchboard/any.go b/e2e/switchboard/any.go similarity index 100% rename from switchboard/any.go rename to e2e/switchboard/any.go diff --git a/switchboard/any_test.go b/e2e/switchboard/any_test.go similarity index 100% rename from switchboard/any_test.go rename to e2e/switchboard/any_test.go diff --git a/switchboard/byID.go b/e2e/switchboard/byID.go similarity index 100% rename from switchboard/byID.go rename to e2e/switchboard/byID.go diff --git a/switchboard/byID_test.go b/e2e/switchboard/byID_test.go similarity index 100% rename from switchboard/byID_test.go rename to e2e/switchboard/byID_test.go diff --git a/switchboard/byType.go b/e2e/switchboard/byType.go similarity index 100% rename from switchboard/byType.go rename to e2e/switchboard/byType.go diff --git a/switchboard/byType_test.go b/e2e/switchboard/byType_test.go similarity index 100% rename from switchboard/byType_test.go rename to e2e/switchboard/byType_test.go diff --git a/switchboard/listener.go b/e2e/switchboard/listener.go similarity index 100% rename from switchboard/listener.go rename to e2e/switchboard/listener.go diff --git a/switchboard/listener_test.go b/e2e/switchboard/listener_test.go similarity index 100% rename from switchboard/listener_test.go rename to e2e/switchboard/listener_test.go diff --git a/e2e/switchboard/message.go b/e2e/switchboard/message.go new file mode 100644 index 0000000000000000000000000000000000000000..e48618843991df0c38867fe840c894f0904edef1 --- /dev/null +++ b/e2e/switchboard/message.go @@ -0,0 +1,23 @@ +package switchboard + +import ( + "gitlab.com/elixxir/client/catalog" + "gitlab.com/elixxir/crypto/e2e" + "gitlab.com/xx_network/primitives/id" + "gitlab.com/xx_network/primitives/id/ephemeral" + "time" +) + +type Receive struct { + MessageType catalog.MessageType + ID e2e.MessageID + Payload []byte + + Sender *id.ID + RecipientID *id.ID + EphemeralID ephemeral.Id + RoundId id.Round + RoundTimestamp time.Time + Timestamp time.Time // Message timestamp of when the user sent + Encrypted bool +} diff --git a/switchboard/switchboard.go b/e2e/switchboard/switchboard.go similarity index 100% rename from switchboard/switchboard.go rename to e2e/switchboard/switchboard.go diff --git a/switchboard/switchboard_test.go b/e2e/switchboard/switchboard_test.go similarity index 100% rename from switchboard/switchboard_test.go rename to e2e/switchboard/switchboard_test.go diff --git a/groupChat/makeGroup.go b/groupChat/makeGroup.go index 944449a6eb9db7ec1082b05279ee554f62467d9a..513ba3643713fd88668ce14a0a46ac91ee3e573d 100644 --- a/groupChat/makeGroup.go +++ b/groupChat/makeGroup.go @@ -10,8 +10,8 @@ package groupChat import ( "github.com/pkg/errors" jww "github.com/spf13/jwalterweatherman" + "gitlab.com/elixxir/client/catalog" gs "gitlab.com/elixxir/client/groupChat/groupStore" - "gitlab.com/elixxir/client/interfaces/preimage" "gitlab.com/elixxir/client/storage/edge" "gitlab.com/elixxir/crypto/contact" "gitlab.com/elixxir/crypto/fastRNG" @@ -97,7 +97,7 @@ func (m Manager) MakeGroup(membership []*id.ID, name, msg []byte) (gs.Group, edgeStore := m.store.GetEdge() edgeStore.Add(edge.Preimage{ Data: g.ID[:], - Type: preimage.Group, + Type: catalog.Group, Source: g.ID[:], }, m.store.GetUser().ReceptionID) } diff --git a/groupChat/manager.go b/groupChat/manager.go index 4b0b3066772f7f7e72e38ef531c32f7d44b3f797..b79631f9553ab792f54380814c9c99527b0062c2 100644 --- a/groupChat/manager.go +++ b/groupChat/manager.go @@ -11,10 +11,10 @@ import ( "github.com/pkg/errors" jww "github.com/spf13/jwalterweatherman" "gitlab.com/elixxir/client/api" + "gitlab.com/elixxir/client/catalog" gs "gitlab.com/elixxir/client/groupChat/groupStore" "gitlab.com/elixxir/client/interfaces" "gitlab.com/elixxir/client/interfaces/message" - "gitlab.com/elixxir/client/interfaces/preimage" "gitlab.com/elixxir/client/stoppable" "gitlab.com/elixxir/client/storage" "gitlab.com/elixxir/client/storage/edge" @@ -134,7 +134,7 @@ func (m Manager) JoinGroup(g gs.Group) error { edgeStore := m.store.GetEdge() edgeStore.Add(edge.Preimage{ Data: g.ID[:], - Type: preimage.Group, + Type: catalog.Group, Source: g.ID[:], }, m.store.GetUser().ReceptionID) @@ -152,7 +152,7 @@ func (m Manager) LeaveGroup(groupID *id.ID) error { edgeStore := m.store.GetEdge() err := edgeStore.Remove(edge.Preimage{ Data: groupID[:], - Type: preimage.Group, + Type: catalog.Group, Source: groupID[:], }, m.store.GetUser().ReceptionID) diff --git a/interfaces/message/receiveMessage.go b/interfaces/message/receiveMessage.go deleted file mode 100644 index 8771a12bd5790cf114229e6a48f5a6714d08000a..0000000000000000000000000000000000000000 --- a/interfaces/message/receiveMessage.go +++ /dev/null @@ -1,28 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// Copyright © 2020 xx network SEZC // -// // -// Use of this source code is governed by a license that can be found in the // -// LICENSE file // -/////////////////////////////////////////////////////////////////////////////// - -package message - -import ( - "gitlab.com/elixxir/crypto/e2e" - "gitlab.com/xx_network/primitives/id" - "gitlab.com/xx_network/primitives/id/ephemeral" - "time" -) - -type Receive struct { - ID e2e.MessageID - Payload []byte - MessageType Type - Sender *id.ID - RecipientID *id.ID - EphemeralID ephemeral.Id - RoundId id.Round - RoundTimestamp time.Time - Timestamp time.Time // Message timestamp of when the user sent - Encryption string -} diff --git a/interfaces/message/sendMessage.go b/interfaces/message/sendMessage.go deleted file mode 100644 index 88795e95ca1aa7a906a2ef953ee059b0fc8af325..0000000000000000000000000000000000000000 --- a/interfaces/message/sendMessage.go +++ /dev/null @@ -1,16 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// Copyright © 2020 xx network SEZC // -// // -// Use of this source code is governed by a license that can be found in the // -// LICENSE file // -/////////////////////////////////////////////////////////////////////////////// - -package message - -import "gitlab.com/xx_network/primitives/id" - -type Send struct { - Recipient *id.ID - Payload []byte - MessageType Type -} diff --git a/interfaces/message/type.go b/interfaces/message/type.go deleted file mode 100644 index 4c444628f46b06a80d12d43485f3c8bf2243ed45..0000000000000000000000000000000000000000 --- a/interfaces/message/type.go +++ /dev/null @@ -1,64 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// Copyright © 2020 xx network SEZC // -// // -// Use of this source code is governed by a license that can be found in the // -// LICENSE file // -/////////////////////////////////////////////////////////////////////////////// - -package message - -const TypeLen = 4 - -type Type uint32 - -const ( - /*general message types*/ - // Used as a wildcard for listeners to listen to all existing types. - // Think of it as "No type in particular" - NoType Type = 0 - - // A message with no message structure - // this is a reserved type, messages sent via SendCmix automatically gain - // this type. Sent messages with this type will be rejected and received - // non Cmix messages will be ignored - Raw Type = 1 - - //Type of message sent by the xx messenger - XxMessage Type = 2 - - /*User Discovery message types*/ - //Message structures defined in the UD package - - // A search for users based on facts. A series of hashed facts are passed - // to UDB - UdSearch = 10 - - // The response to the UD search. It contains a list of contact objects - // matching the sent facts - UdSearchResponse = 11 - - // Searched for the DH public key associated with the passed User ID - UdLookup = 12 - - // Response to UdLookup, it contains the associated public key if one is - // available - UdLookupResponse = 13 - - /*End to End Rekey message types*/ - // Trigger a rekey, this message is used locally in client only - KeyExchangeTrigger = 30 - // Rekey confirmation message. Sent by partner to confirm completion of a rekey - KeyExchangeConfirm = 31 - - /* Group chat message types */ - // A group chat request message sent to all members in a group. - GroupCreationRequest = 40 - - // NewFileTransfer is transmitted first on the initialization of a file - // transfer to inform the receiver about the incoming file. - NewFileTransfer = 50 - - // EndFileTransfer is sent once all file parts have been transmitted to - // inform the receiver that the file transfer has ended. - EndFileTransfer = 51 -) diff --git a/interfaces/preimage/generate.go b/interfaces/preimage/generate.go deleted file mode 100644 index cf176599482c0f448638fdbad9c7bbdfc4fe2421..0000000000000000000000000000000000000000 --- a/interfaces/preimage/generate.go +++ /dev/null @@ -1,29 +0,0 @@ -package preimage - -import ( - "gitlab.com/xx_network/primitives/id" - "golang.org/x/crypto/blake2b" -) - -func Generate(data []byte, t string) []byte { - if t == Default { - return data - } - // Hash fingerprints - h, _ := blake2b.New256(nil) - h.Write(data) - h.Write([]byte(t)) - - // Base 64 encode hash and truncate - return h.Sum(nil) -} - -func GenerateRequest(recipient *id.ID) []byte { - // Hash fingerprints - h, _ := blake2b.New256(nil) - h.Write(recipient[:]) - h.Write([]byte(Request)) - - // Base 64 encode hash and truncate - return h.Sum(nil) -} diff --git a/interfaces/preimage/request.go b/interfaces/preimage/request.go deleted file mode 100644 index 0eb0689362e92e1f3145572f2551a45448adef54..0000000000000000000000000000000000000000 --- a/interfaces/preimage/request.go +++ /dev/null @@ -1,20 +0,0 @@ -package preimage - -import ( - "gitlab.com/xx_network/primitives/id" - "golang.org/x/crypto/blake2b" -) - -func MakeRequest(uid *id.ID) []byte { - h, _ := blake2b.New256(nil) - h.Write(uid[:]) - h.Write([]byte(Request)) - - // Base 64 encode hash and truncate - return h.Sum(nil) -} - -func MakeDefault(uid *id.ID) []byte { - // Base 64 encode hash and truncate - return uid[:] -} diff --git a/network/historical/historical.go b/network/historical/historical.go index 37f4fdc5ec64de5da1ec79805548aae4f5d04bdd..5ed89c1cd4708aa877d53bc51f9cb04bea2be25f 100644 --- a/network/historical/historical.go +++ b/network/historical/historical.go @@ -53,7 +53,7 @@ type RoundsComms interface { } // RoundResultCallback is the used callback when a round is found. -type RoundResultCallback func(info *pb.RoundInfo, success bool) +type RoundResultCallback func(info Round, success bool) // roundRequest is an internal structure that tracks a request. type roundRequest struct { @@ -217,7 +217,7 @@ func processHistoricalRoundsResponse(response *pb.HistoricalRoundsResponse, if roundRequests[i].numAttempts == maxRetries { errMsg = fmt.Sprintf("Failed to retrieve historical round %d "+ "on last attempt, will not try again", roundRequests[i].rid) - go roundRequests[i].RoundResultCallback(nil, false) + go roundRequests[i].RoundResultCallback(Round{}, false) } else { retries = append(retries, roundRequests[i]) errMsg = fmt.Sprintf("Failed to retrieve historical round "+ @@ -231,7 +231,7 @@ func processHistoricalRoundsResponse(response *pb.HistoricalRoundsResponse, } // Successfully retrieved roundRequests are returned on the callback - go roundRequests[i].RoundResultCallback(roundInfo, true) + go roundRequests[i].RoundResultCallback(MakeRound(roundInfo), true) rids = append(rids, roundInfo.ID) } diff --git a/network/historical/historical_test.go b/network/historical/historical_test.go index 676fb1c75ef0b47ff980325ddf01c411509c8ec7..7301dc5d9832a23e3ecd3df5c5d1160926862f56 100644 --- a/network/historical/historical_test.go +++ b/network/historical/historical_test.go @@ -32,7 +32,7 @@ func TestHistoricalRounds(t *testing.T) { stopper := hMgr.StartProcesses() // Case 1: Send a round request and wait for timeout for processing - err := hMgr.LookupHistoricalRound(42, func(*pb.RoundInfo, bool) { + err := hMgr.LookupHistoricalRound(42, func(Round, bool) { t.Error("Called when it should not have been.") }) if err != nil { @@ -46,7 +46,7 @@ func TestHistoricalRounds(t *testing.T) { // Case 2: make round requests up to m.params.MaxHistoricalRounds for i := id.Round(0); i < 3; i++ { - err = hMgr.LookupHistoricalRound(40+i, func(*pb.RoundInfo, bool) { + err = hMgr.LookupHistoricalRound(40+i, func(Round, bool) { t.Errorf("%d called when it should not have been.", i) }) if err != nil { @@ -74,15 +74,15 @@ func TestProcessHistoricalRoundsResponse(t *testing.T) { params := GetDefaultParams() badRR := roundRequest{ rid: id.Round(41), - RoundResultCallback: func(*pb.RoundInfo, bool) { + RoundResultCallback: func(Round, bool) { t.Error("Called when it should not have been.") }, numAttempts: params.MaxHistoricalRoundsRetries - 2, } expiredRR := roundRequest{ rid: id.Round(42), - RoundResultCallback: func(info *pb.RoundInfo, success bool) { - if info == nil && !success { + RoundResultCallback: func(info Round, success bool) { + if info.ID == 0 && !success { return } t.Errorf("Expired called with bad params.") @@ -93,7 +93,7 @@ func TestProcessHistoricalRoundsResponse(t *testing.T) { callbackCalled := &x goodRR := roundRequest{ rid: id.Round(43), - RoundResultCallback: func(info *pb.RoundInfo, success bool) { + RoundResultCallback: func(info Round, success bool) { *callbackCalled = true }, numAttempts: 0, diff --git a/network/historical/round.go b/network/historical/round.go new file mode 100644 index 0000000000000000000000000000000000000000..f7b7397196944a1486328506447167fee330b3b4 --- /dev/null +++ b/network/historical/round.go @@ -0,0 +1,101 @@ +package historical + +import ( + pb "gitlab.com/elixxir/comms/mixmessages" + "gitlab.com/elixxir/primitives/states" + "gitlab.com/xx_network/comms/connect" + "gitlab.com/xx_network/primitives/id" + "time" +) + +type Round struct { + //ID of the round. Ids are sequential and monotonic + ID id.Round + + // Last known state of the round. Possible states are: + // PENDING - not started yet + // PRECOMPUTING - In the process of preparing to process messages + // STANDBY - Completed precomputing but not yet scheduled to run + // QUEUED - Scheduled to run at a set time + // REALTIME - Running, actively handing messages + // COMPLETED - Successfully deleted messages + // FAILED - Failed to deliver messages + State states.Round + + // List of Nodes in the round + Topology *connect.Circuit + + // Timestamps of all events that have occurred in the round + // (See the above states). + // The Queued state's timestamp is different, it denotes when Realtime + // was/is scheduled to start, not whe the Queued state is entered + Timestamps map[states.Round]time.Time + + // Errors that occurred in the round. Will only be present in the failed + // state + Errors []RoundError + + /*Properties*/ + + // Max number of messages the round can process + BatchSize uint32 + + // Ephemeral Address space size used in the round + AddressSpaceSize uint8 + + // Monotonic counter between all round updates denoting when this updated + //occurred in the queue + UpdateID uint64 + + // RawData round data, including signatures + Raw *pb.RoundInfo +} + +type RoundError struct { + NodeID *id.ID + Error string +} + +//MakeRound Builds an accessable round object from a RoundInfo Protobuff +func MakeRound(ri *pb.RoundInfo) Round { + //Build the timestamps map + timestamps := make(map[states.Round]time.Time) + + for i := range ri.Timestamps { + if ri.Timestamps[i] != 0 { + timestamps[states.Round(i)] = + time.Unix(0, int64(ri.Timestamps[i])) + } + } + + //Build the input to the topology + nodes := make([]*id.ID, len(ri.Topology)) + for i := range ri.Topology { + newNodeID := id.ID{} + copy(newNodeID[:], ri.Topology[i]) + nodes[i] = &newNodeID + } + + //build the errors + errs := make([]RoundError, len(ri.Errors)) + for i := range ri.Errors { + errNodeID := id.ID{} + copy(errNodeID[:], ri.Errors[i].NodeId) + errs[i] = RoundError{ + NodeID: &errNodeID, + Error: ri.Errors[i].Error, + } + } + + return Round{ + ID: id.Round(ri.ID), + State: states.Round(ri.State), + Topology: connect.NewCircuit(nodes), + Timestamps: timestamps, + Errors: errs, + BatchSize: ri.BatchSize, + AddressSpaceSize: uint8(ri.AddressSpaceSize), + UpdateID: ri.UpdateID, + Raw: ri, + } +} diff --git a/network/message/bundle.go b/network/message/bundle.go index a3fe68d783b106475ca9d79f77ac56234918c9e5..13b1a619a1be45c60d3c73c495b696b79a76f73c 100644 --- a/network/message/bundle.go +++ b/network/message/bundle.go @@ -8,15 +8,15 @@ package message import ( + "gitlab.com/elixxir/client/network/historical" "gitlab.com/elixxir/client/network/identity/receptionID" - pb "gitlab.com/elixxir/comms/mixmessages" "gitlab.com/elixxir/primitives/format" "gitlab.com/xx_network/primitives/id" ) type Bundle struct { Round id.Round - RoundInfo *pb.RoundInfo + RoundInfo historical.Round Messages []format.Message Finish func() Identity receptionID.EphemeralIdentity diff --git a/network/message/fingerprints_test.go b/network/message/fingerprints_test.go index 40670107aa30847b9005da3a25817b7f3c0f3a6b..84a38dda943ba5108e9da58115b8ea7de49ead04 100644 --- a/network/message/fingerprints_test.go +++ b/network/message/fingerprints_test.go @@ -9,8 +9,8 @@ package message import ( jww "github.com/spf13/jwalterweatherman" + "gitlab.com/elixxir/client/network/historical" "gitlab.com/elixxir/client/network/identity/receptionID" - "gitlab.com/elixxir/comms/mixmessages" "gitlab.com/elixxir/primitives/format" "gitlab.com/xx_network/primitives/id" "reflect" @@ -173,6 +173,6 @@ func (mock *MockMsgProcessor) MarkFingerprintUsed(_ format.Fingerprint) { } func (mock *MockMsgProcessor) Process(format.Message, receptionID.EphemeralIdentity, - *mixmessages.RoundInfo) { + historical.Round) { return } diff --git a/network/message/handler.go b/network/message/handler.go index 26ca4d8b1e640ec5c1cba84f1ba16bd411a680a9..dad3888b9c29f8bae2c19ae4c04b93147c1d60c5 100644 --- a/network/message/handler.go +++ b/network/message/handler.go @@ -31,12 +31,12 @@ func (p *handler) handleMessages(stop *stoppable.Single) { go func() { count, ts := p.inProcess.Add( - msg, bundle.RoundInfo, bundle.Identity) + msg, bundle.RoundInfo.Raw, bundle.Identity) wg.Done() success := p.handleMessage(msg, bundle) if success { p.inProcess.Remove( - msg, bundle.RoundInfo, bundle.Identity) + msg, bundle.RoundInfo.Raw, bundle.Identity) } else { // Fail the message if any part of the decryption // fails, unless it is the last attempts and has @@ -45,10 +45,10 @@ func (p *handler) handleMessages(stop *stoppable.Single) { if count == p.param.MaxChecksInProcessMessage && netTime.Since(ts) > p.param.InProcessMessageWait { p.inProcess.Remove( - msg, bundle.RoundInfo, bundle.Identity) + msg, bundle.RoundInfo.Raw, bundle.Identity) } else { p.inProcess.Failed( - msg, bundle.RoundInfo, bundle.Identity) + msg, bundle.RoundInfo.Raw, bundle.Identity) } } diff --git a/network/message/inProgress.go b/network/message/inProgress.go index dd28e71e9355d82713d08383e25bdddab0a665d7..c3f58fed8d2d0df847ef8c3268dca21b3c52c6ad 100644 --- a/network/message/inProgress.go +++ b/network/message/inProgress.go @@ -9,6 +9,7 @@ package message import ( jww "github.com/spf13/jwalterweatherman" + "gitlab.com/elixxir/client/network/historical" "gitlab.com/elixxir/client/stoppable" "gitlab.com/elixxir/primitives/format" "gitlab.com/xx_network/primitives/id" @@ -53,7 +54,7 @@ func (p *handler) recheckInProgress() { for grbldMsg, ri, identity, has := p.inProcess.Next(); has; grbldMsg, ri, identity, has = p.inProcess.Next() { bundle := Bundle{ Round: id.Round(ri.ID), - RoundInfo: ri, + RoundInfo: historical.MakeRound(ri), Messages: []format.Message{grbldMsg}, Finish: func() {}, Identity: identity, diff --git a/network/message/processor.go b/network/message/processor.go index 214c845f307c7ef978d98ba08cbf8836877acc07..1e316fb161006c021d7322d65ffc03dca5cb2d73 100644 --- a/network/message/processor.go +++ b/network/message/processor.go @@ -1,8 +1,8 @@ package message import ( + "gitlab.com/elixxir/client/network/historical" "gitlab.com/elixxir/client/network/identity/receptionID" - "gitlab.com/elixxir/comms/mixmessages" "gitlab.com/elixxir/primitives/format" ) @@ -15,5 +15,5 @@ type Processor interface { // It is a security vulnerability to reuse a fingerprint. It leaks privacy // and can lead to compromise of message contents and integrity. Process(message format.Message, receptionID receptionID.EphemeralIdentity, - round *mixmessages.RoundInfo) + round historical.Round) } diff --git a/network/rounds/get.go b/network/rounds/get.go index 7fd430d19a0053f5b3d3fd838b5d7b5fa7bc797c..56a7f1638ef0c57a3136a6f75f33c376282c5d12 100644 --- a/network/rounds/get.go +++ b/network/rounds/get.go @@ -9,8 +9,8 @@ package rounds import ( jww "github.com/spf13/jwalterweatherman" + "gitlab.com/elixxir/client/network/historical" "gitlab.com/elixxir/client/network/identity/receptionID" - pb "gitlab.com/elixxir/comms/mixmessages" "gitlab.com/xx_network/primitives/id" ) @@ -40,14 +40,14 @@ func (m *manager) GetMessagesFromRound(roundID id.Round, identity receptionID.Ep identity.Source) err = m.historical.LookupHistoricalRound( - roundID, func(info *pb.RoundInfo, success bool) { + roundID, func(info historical.Round, success bool) { if !success { // TODO: implement me } // If found, send to Message Retrieval Workers m.lookupRoundMessages <- roundLookup{ - RoundInfo: info, - Identity: identity, + Round: info, + Identity: identity, } }) } else { @@ -68,8 +68,8 @@ func (m *manager) GetMessagesFromRound(roundID id.Round, identity receptionID.Ep // If found, send to Message Retrieval Workers m.lookupRoundMessages <- roundLookup{ - RoundInfo: ri, - Identity: identity, + Round: historical.MakeRound(ri), + Identity: identity, } } diff --git a/network/rounds/retrieve.go b/network/rounds/retrieve.go index 5891480cc5a1b84e8351c6a461417f63205244b3..dcf1917d4cef8972be3168708c8f971f883bd063 100644 --- a/network/rounds/retrieve.go +++ b/network/rounds/retrieve.go @@ -12,6 +12,7 @@ import ( "github.com/pkg/errors" jww "github.com/spf13/jwalterweatherman" "gitlab.com/elixxir/client/network/gateway" + "gitlab.com/elixxir/client/network/historical" "gitlab.com/elixxir/client/network/identity/receptionID" "gitlab.com/elixxir/client/network/message" "gitlab.com/elixxir/client/stoppable" @@ -30,8 +31,8 @@ type MessageRetrievalComms interface { } type roundLookup struct { - RoundInfo *pb.RoundInfo - Identity receptionID.EphemeralIdentity + Round historical.Round + Identity receptionID.EphemeralIdentity } const noRoundError = "does not have round %d" @@ -47,11 +48,11 @@ func (m *manager) processMessageRetrieval(comms MessageRetrievalComms, stop.ToStopped() return case rl := <-m.lookupRoundMessages: - ri := rl.RoundInfo + ri := rl.Round jww.DEBUG.Printf("Checking for messages in round %d", ri.ID) if !m.params.RealtimeOnly { - err := m.unchecked.AddRound(id.Round(ri.ID), ri, + err := m.unchecked.AddRound(id.Round(ri.ID), ri.Raw, rl.Identity.Source, rl.Identity.EphId) if err != nil { jww.FATAL.Panicf( @@ -61,13 +62,9 @@ func (m *manager) processMessageRetrieval(comms MessageRetrievalComms, } // Convert gateways in round to proper ID format - gwIds := make([]*id.ID, len(ri.Topology)) - for i, idBytes := range ri.Topology { - gwId, err := id.Unmarshal(idBytes) - if err != nil { - jww.FATAL.Panicf( - "processMessageRetrieval: Unable to unmarshal: %+v", err) - } + gwIds := make([]*id.ID, ri.Topology.Len()) + for i := 0; i < ri.Topology.Len(); i++ { + gwId := ri.Topology.GetNodeAtIndex(i).DeepCopy() gwId.SetType(id.Gateway) gwIds[i] = gwId } @@ -139,7 +136,7 @@ func (m *manager) processMessageRetrieval(comms MessageRetrievalComms, EphId: rl.Identity.EphId, Source: rl.Identity.Source, } - bundle.RoundInfo = rl.RoundInfo + bundle.RoundInfo = rl.Round m.messageBundles <- bundle jww.DEBUG.Printf("Removing round %d from unchecked store", ri.ID) @@ -249,11 +246,11 @@ func (m *manager) getMessagesFromGateway(roundID id.Round, // Helper function which forces processUncheckedRounds by randomly not looking // up messages. -func (m *manager) forceMessagePickupRetry(ri *pb.RoundInfo, rl roundLookup, +func (m *manager) forceMessagePickupRetry(ri historical.Round, rl roundLookup, comms MessageRetrievalComms, gwIds []*id.ID, stop *stoppable.Single) (bundle message.Bundle, err error) { rnd, _ := m.unchecked.GetRound( - id.Round(ri.ID), rl.Identity.Source, rl.Identity.EphId) + ri.ID, rl.Identity.Source, rl.Identity.EphId) if rnd.NumChecks == 0 { // Flip a coin to determine whether to pick up message b := make([]byte, 8) @@ -276,5 +273,5 @@ func (m *manager) forceMessagePickupRetry(ri *pb.RoundInfo, rl roundLookup, // Attempt to request for this gateway return m.getMessagesFromGateway( - id.Round(ri.ID), rl.Identity, comms, gwIds, stop) + ri.ID, rl.Identity, comms, gwIds, stop) } diff --git a/network/rounds/retrieve_test.go b/network/rounds/retrieve_test.go index 1d5a5225e88587103e03d1d2b4138bab13009b80..744fe23e48d9eb6c93828c1397c367e9e0fd6528 100644 --- a/network/rounds/retrieve_test.go +++ b/network/rounds/retrieve_test.go @@ -9,10 +9,10 @@ package rounds import ( "bytes" "gitlab.com/elixxir/client/network/gateway" + "gitlab.com/elixxir/client/network/historical" ephemeral2 "gitlab.com/elixxir/client/network/identity/receptionID" "gitlab.com/elixxir/client/network/message" "gitlab.com/elixxir/client/stoppable" - pb "gitlab.com/elixxir/comms/mixmessages" "gitlab.com/elixxir/crypto/fastRNG" "gitlab.com/xx_network/comms/connect" "gitlab.com/xx_network/crypto/csprng" @@ -71,17 +71,15 @@ func TestManager_ProcessMessageRetrieval(t *testing.T) { Source: requestGateway, } - idList := [][]byte{requestGateway.Bytes()} - - roundInfo := &pb.RoundInfo{ - ID: uint64(roundId), - Topology: idList, + roundInfo := historical.Round{ + ID: roundId, + Topology: connect.NewCircuit([]*id.ID{requestGateway}), } // Send a round look up request testManager.lookupRoundMessages <- roundLookup{ - RoundInfo: roundInfo, - Identity: ephIdentity, + Round: roundInfo, + Identity: ephIdentity, } }() @@ -162,17 +160,15 @@ func TestManager_ProcessMessageRetrieval_NoRound(t *testing.T) { Source: dummyGateway, } - idList := [][]byte{dummyGateway.Marshal()} - - roundInfo := &pb.RoundInfo{ - ID: uint64(roundId), - Topology: idList, + roundInfo := historical.Round{ + ID: roundId, + Topology: connect.NewCircuit([]*id.ID{dummyGateway}), } // Send a round look up request testManager.lookupRoundMessages <- roundLookup{ - RoundInfo: roundInfo, - Identity: identity, + Round: roundInfo, + Identity: identity, } }() @@ -241,17 +237,15 @@ func TestManager_ProcessMessageRetrieval_FalsePositive(t *testing.T) { requestGateway := id.NewIdFromString(FalsePositive, id.Gateway, t) - idList := [][]byte{requestGateway.Bytes()} - - roundInfo := &pb.RoundInfo{ - ID: uint64(roundId), - Topology: idList, + roundInfo := historical.Round{ + ID: roundId, + Topology: connect.NewCircuit([]*id.ID{requestGateway}), } // Send a round look up request testManager.lookupRoundMessages <- roundLookup{ - RoundInfo: roundInfo, - Identity: identity, + Round: roundInfo, + Identity: identity, } }() @@ -316,17 +310,14 @@ func TestManager_ProcessMessageRetrieval_Quit(t *testing.T) { requestGateway := id.NewIdFromString(ReturningGateway, id.Gateway, t) - idList := [][]byte{requestGateway.Bytes()} - - roundInfo := &pb.RoundInfo{ - ID: uint64(roundId), - Topology: idList, + roundInfo := historical.Round{ + ID: roundId, + Topology: connect.NewCircuit([]*id.ID{requestGateway}), } - // Send a round look up request testManager.lookupRoundMessages <- roundLookup{ - RoundInfo: roundInfo, - Identity: identity, + Round: roundInfo, + Identity: identity, } }() @@ -390,18 +381,18 @@ func TestManager_ProcessMessageRetrieval_MultipleGateways(t *testing.T) { Source: requestGateway, } - // Create a list of ID's in which some error gateways must be contacted before the happy path - idList := [][]byte{errorGateway.Bytes(), errorGateway.Bytes(), requestGateway.Bytes()} - - roundInfo := &pb.RoundInfo{ - ID: uint64(roundId), - Topology: idList, + roundInfo := historical.Round{ + ID: roundId, + // Create a list of ID's in which some error gateways must be + //contacted before the happy path + Topology: connect.NewCircuit([]*id.ID{errorGateway, errorGateway, + requestGateway}), } // Send a round look up request testManager.lookupRoundMessages <- roundLookup{ - RoundInfo: roundInfo, - Identity: identity, + Round: roundInfo, + Identity: identity, } }() diff --git a/storage/edge/edge_test.go b/storage/edge/edge_test.go index c7823245f47a29d839622a63200ea46dfcba9185..3e00575002939489077e0072223750f241dd577e 100644 --- a/storage/edge/edge_test.go +++ b/storage/edge/edge_test.go @@ -16,6 +16,7 @@ package edge import ( "encoding/json" + "gitlab.com/elixxir/client/catalog" "gitlab.com/elixxir/client/interfaces/preimage" "gitlab.com/elixxir/client/storage/versioned" fingerprint2 "gitlab.com/elixxir/crypto/fingerprint" @@ -149,7 +150,7 @@ func TestStore_Add(t *testing.T) { "\nexpected: %d\nreceived: %d", identities[0], 3, len(pis)) } - expected := Preimage{preimage.MakeDefault(identities[0]), preimage.Default, identities[0].Bytes()} + expected := Preimage{preimage.MakeDefault(identities[0]), catalog.Default, identities[0].Bytes()} if !reflect.DeepEqual(pis[expected.key()], expected) { t.Errorf("First Preimage of first Preimages does not match expected."+ "\nexpected: %+v\nreceived: %+v", expected, pis[expected.key()]) @@ -174,7 +175,7 @@ func TestStore_Add(t *testing.T) { "\nexpected: %d\nreceived: %d", identities[1], 2, len(pis)) } - expected = Preimage{preimage.MakeDefault(identities[1]), preimage.Default, identities[1].Bytes()} + expected = Preimage{preimage.MakeDefault(identities[1]), catalog.Default, identities[1].Bytes()} if !reflect.DeepEqual(pis[expected.key()], expected) { t.Errorf("First Preimage of second Preimages does not match expected."+ "\nexpected: %+v\nreceived: %+v", expected, pis[expected.key()]) @@ -349,7 +350,7 @@ func TestStore_Get(t *testing.T) { } expected := []Preimage{ - {preimage.MakeDefault(identities[0]), preimage.Default, identities[0].Bytes()}, + {preimage.MakeDefault(identities[0]), catalog.Default, identities[0].Bytes()}, preimages[0], preimages[2], } @@ -375,7 +376,7 @@ top: } expected = []Preimage{ - {preimage.MakeDefault(identities[1]), preimage.Default, identities[1].Bytes()}, + {preimage.MakeDefault(identities[1]), catalog.Default, identities[1].Bytes()}, preimages[1], } @@ -475,12 +476,12 @@ func TestLoadStore(t *testing.T) { expectedPis := [][]Preimage{ { - Preimage{preimage.MakeDefault(identities[0]), preimage.Default, identities[0].Bytes()}, + Preimage{preimage.MakeDefault(identities[0]), catalog.Default, identities[0].Bytes()}, preimages[0], preimages[2], }, { - Preimage{preimage.MakeDefault(identities[1]), preimage.Default, identities[1].Bytes()}, + Preimage{preimage.MakeDefault(identities[1]), catalog.Default, identities[1].Bytes()}, preimages[1], }, } diff --git a/storage/edge/preimage.go b/storage/edge/preimage.go index f6d62432f81db03232f4e94aaea756eb6ac96642..9eca518bbde416f2c4b9d65fd06004a1aae3bb0e 100644 --- a/storage/edge/preimage.go +++ b/storage/edge/preimage.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/json" "github.com/pkg/errors" + "gitlab.com/elixxir/client/catalog" "gitlab.com/elixxir/client/interfaces/preimage" "gitlab.com/elixxir/client/storage/versioned" "gitlab.com/xx_network/primitives/id" @@ -34,7 +35,7 @@ type Preimages map[string]Preimage func newPreimages(identity *id.ID) Preimages { defaultPreimage := Preimage{ Data: preimage.MakeDefault(identity), - Type: preimage.Default, + Type: catalog.Default, Source: identity[:], } pis := Preimages{ diff --git a/storage/edge/preimage_test.go b/storage/edge/preimage_test.go index b31cc4274075344ad705b5308e7ac30e93bac5d0..7634b11832a75bbaf80fcefd6a3da754a64ded4b 100644 --- a/storage/edge/preimage_test.go +++ b/storage/edge/preimage_test.go @@ -10,6 +10,7 @@ package edge import ( "bytes" "encoding/json" + "gitlab.com/elixxir/client/catalog" "gitlab.com/elixxir/client/interfaces/preimage" "gitlab.com/elixxir/client/storage/versioned" "gitlab.com/elixxir/ekv" @@ -44,19 +45,19 @@ func TestPreimages_add(t *testing.T) { identity1 := id.NewIdFromString("identity1", id.User, t) identity2 := id.NewIdFromString("identity3", id.User, t) expected := Preimages{ - identity0.String(): {preimage.Generate(identity0.Bytes(), preimage.Default), preimage.Default, preimage.MakeDefault(identity0)}, - identity1.String(): {preimage.Generate(identity1.Bytes(), preimage.Group), preimage.Group, identity1.Bytes()}, - identity2.String(): {preimage.Generate(identity2.Bytes(), preimage.Default), preimage.Default, identity2.Bytes()}, + identity0.String(): {preimage.Generate(identity0.Bytes(), catalog.Default), catalog.Default, preimage.MakeDefault(identity0)}, + identity1.String(): {preimage.Generate(identity1.Bytes(), catalog.Group), catalog.Group, identity1.Bytes()}, + identity2.String(): {preimage.Generate(identity2.Bytes(), catalog.Default), catalog.Default, identity2.Bytes()}, } pis := newPreimages(identity0) - preimageOne := Preimage{preimage.Generate(identity1.Bytes(), preimage.Group), preimage.Group, identity1.Bytes()} + preimageOne := Preimage{preimage.Generate(identity1.Bytes(), catalog.Group), catalog.Group, identity1.Bytes()} exists := pis.add(preimageOne) if !exists { t.Errorf("Failed to add idenetity.") } - preimageTwo := Preimage{preimage.Generate(identity2.Bytes(), preimage.Default), preimage.Default, identity2.Bytes()} + preimageTwo := Preimage{preimage.Generate(identity2.Bytes(), catalog.Default), catalog.Default, identity2.Bytes()} exists = pis.add(preimageTwo) if !exists { t.Errorf("Failed to add idenetity.") @@ -69,12 +70,12 @@ func TestPreimages_add(t *testing.T) { } expectedPreimageIdentityTwo := Preimage{ - Data: preimage.Generate(identity2.Bytes(), preimage.Default), - Type: preimage.Default, + Data: preimage.Generate(identity2.Bytes(), catalog.Default), + Type: catalog.Default, Source: identity2.Bytes(), } // Test that nothing happens when a Preimage with the same data exists - exists = pis.add(Preimage{preimage.Generate(identity2.Bytes(), preimage.Default), "test", identity2.Bytes()}) + exists = pis.add(Preimage{preimage.Generate(identity2.Bytes(), catalog.Default), "test", identity2.Bytes()}) if exists { t.Errorf("Add idenetity that shoudl already exist.") } @@ -95,9 +96,9 @@ func TestPreimages_remove(t *testing.T) { // Add 10 Preimage to the list for i := 0; i < 10; i++ { identity := id.NewIdFromUInt(uint64(i), id.User, t) - pisType := preimage.Default + pisType := catalog.Default if i%2 == 0 { - pisType = preimage.Group + pisType = catalog.Group } exists := pis.add(Preimage{identity.Bytes(), pisType, identity.Bytes()})