fixed issue with ephemeral IDs, both deletion and thread handling, within the receptionID package

cmix/follow.go

@@ -87,43 +87,50 @@ func (c *client) followNetwork(report ClientErrorReport,
 			stop.ToStopped()
 			return
 		case <-ticker.C:
+
+			operator := func(toTrack []receptionID.IdentityUse) error {
+
+				// set up tracking tools
+				wg := &sync.WaitGroup{}
+				wg.Add(len(toTrack))
+
+				// trigger the first separately because it will get network state
+				// updates
+				go func() {
+					c.follow(toTrack[0], report, rng, c.comms, stop, abandon,
+						true)
+					wg.Done()
+				}()
+
+				//trigger all others without getting network state updates
+				for i := 1; i < len(toTrack); i++ {
+					go func(index int) {
+						c.follow(toTrack[index], report, rng, c.comms, stop,
+							dummyAbandon, false)
+						wg.Done()
+					}(i)
+				}
+
+				//wait for all to complete
+				wg.Wait()
+				return nil
+			}
+
 			// get the list of identities to track
 			stream := c.rng.GetStream()
-			toTrack, err := c.Tracker.GetEphemeralIdentities(
+			err := c.Tracker.ForEach(
 				int(c.param.MaxParallelIdentityTracks),
 				stream,
-				c.Space.GetAddressSpaceWithoutWait())
+				c.Space.GetAddressSpaceWithoutWait(),
+				operator)
 			stream.Close()
 
 			if err != nil {
-				jww.ERROR.Printf("failed to get identities to track")
+				jww.ERROR.Printf("failed to operate on identities to "+
+					"track: %s", err)
 				continue
 			}
 
-			// set up tracking tools
-			wg := &sync.WaitGroup{}
-			wg.Add(len(toTrack))
-
-			// trigger the first separately because it will get network state
-			// updates
-			go func() {
-				c.follow(toTrack[0], report, rng, c.comms, stop, abandon,
-					true)
-				wg.Done()
-			}()
-
-			//trigger all others without getting network state updates
-			for i := 1; i < len(toTrack); i++ {
-				go func(index int) {
-					c.follow(toTrack[index], report, rng, c.comms, stop,
-						dummyAbandon, false)
-					wg.Done()
-				}(i)
-			}
-
-			//wait for all to complete
-			wg.Wait()
-
 		case <-TrackTicker.C:
 			numPolls := atomic.SwapUint64(c.tracker, 0)
 			if c.numLatencies != 0 {

cmix/identity/receptionID/store.go

@@ -160,46 +160,13 @@ func (s *Store) makeStoredReferences() []storedReference {
 	return identities[:i]
 }
 
-// GetIdentity will return a single identity. If none are available, it will
-// return a fake one
-func (s *Store) GetIdentity(rng io.Reader, addressSize uint8) IdentityUse {
-	s.mux.Lock()
-	defer s.mux.Unlock()
-
-	now := netTime.Now()
-
-	// Remove any now expired identities
-	s.prune(now)
-
-	var identity IdentityUse
-	var err error
-
-	// If the list is empty, then return a randomly generated identity to poll
-	// with so that we can continue tracking the network and to further
-	// obfuscate network identities.
-	if len(s.active) == 0 {
-		identity, err = generateFakeIdentity(rng, addressSize, now)
-		if err != nil {
-			jww.FATAL.Panicf(
-				"Failed to generate a new ID when none available: %+v", err)
-		}
-	} else {
-		identity, err = s.selectIdentity(rng, now)
-		if err != nil {
-			jww.FATAL.Panicf("Failed to select an ID: %+v", err)
-		}
-	}
-
-	return identity
-}
-
-// GetIdentities will return up to 'n' identities randomly in a random order.
-// if no identities exist, it will return a single fake identity
-func (s *Store) GetIdentities(n int, rng io.Reader,
-	addressSize uint8) ([]IdentityUse, error) {
+// ForEach operates on 'n' identities randomly in a random order.
+// if no identities exist, it will operate on a single fake identity
+func (s *Store) ForEach(n int, rng io.Reader,
+	addressSize uint8, operate func([]IdentityUse) error) error {
 
 	if n < 1 {
-		return nil, InvalidRequestedNumIdentities
+		return InvalidRequestedNumIdentities
 	}
 
 	s.mux.Lock()
@@ -231,7 +198,8 @@ func (s *Store) GetIdentities(n int, rng io.Reader,
 		}
 	}
 
-	return identities, nil
+	// do the passed operation on all identities
+	return operate(identities)
 }
 
 func (s *Store) AddIdentity(identity Identity) error {
@@ -274,7 +242,8 @@ func (s *Store) RemoveIdentity(ephID ephemeral.Id) {
 	s.mux.Lock()
 	defer s.mux.Unlock()
 
-	for i, inQuestion := range s.active {
+	for i := 0; i < len(s.active); i++ {
+		inQuestion := s.active[i]
 		if inQuestion.EphId == ephID {
 			s.active = append(s.active[:i], s.active[i+1:]...)
 
@@ -290,6 +259,8 @@ func (s *Store) RemoveIdentity(ephID ephemeral.Id) {
 				}
 			}
 
+			i--
+
 			return
 		}
 	}
@@ -300,7 +271,8 @@ func (s *Store) RemoveIdentities(source *id.ID) {
 	defer s.mux.Unlock()
 
 	doSave := false
-	for i, inQuestion := range s.active {
+	for i := 0; i < len(s.active); i++ {
+		inQuestion := s.active[i]
 		if inQuestion.Source.Cmp(source) {
 			s.active = append(s.active[:i], s.active[i+1:]...)
 
@@ -310,6 +282,7 @@ func (s *Store) RemoveIdentities(source *id.ID) {
 			}
 
 			doSave = doSave || !inQuestion.Ephemeral
+			i--
 		}
 	}
 	if doSave {

cmix/identity/receptionID/store_test.go

@@ -151,38 +151,6 @@ func TestStore_makeStoredReferences(t *testing.T) {
 	}
 }
 
-func TestStore_GetIdentity(t *testing.T) {
-	kv := versioned.NewKV(ekv.MakeMemstore())
-	s := NewOrLoadStore(kv)
-	prng := rand.New(rand.NewSource(42))
-	testID, err := generateFakeIdentity(prng, 15, netTime.Now())
-	if err != nil {
-		t.Fatalf("Failed to generate fake ID: %+v", err)
-	}
-	if s.AddIdentity(testID.Identity) != nil {
-		t.Errorf("AddIdentity() produced an error: %+v", err)
-	}
-
-	idu := s.GetIdentity(prng, 15)
-
-	if !testID.Equal(idu.Identity) {
-		t.Errorf("GetIdentity() did not return the expected Identity."+
-			"\nexpected: %s\nreceived: %s", testID, idu)
-	}
-}
-
-func TestStore_GetIdentity_NoIdentities(t *testing.T) {
-	kv := versioned.NewKV(ekv.MakeMemstore())
-	s := NewOrLoadStore(kv)
-	prng := rand.New(rand.NewSource(42))
-
-	idu := s.GetIdentity(prng, 15)
-
-	if !idu.Fake {
-		t.Errorf("GetIdentity() did not return a fake identity")
-	}
-}
-
 func TestStore_GetIdentities(t *testing.T) {
 	kv := versioned.NewKV(ekv.MakeMemstore())
 	s := NewOrLoadStore(kv)
@@ -207,7 +175,12 @@ func TestStore_GetIdentities(t *testing.T) {
 	}
 
 	//get one
-	idu, err := s.GetIdentities(1, prng, 15)
+	var idu []IdentityUse
+	o := func(a []IdentityUse) error {
+		idu = a
+		return nil
+	}
+	err := s.ForEach(1, prng, 15, o)
 	if err != nil {
 		t.Errorf("GetIdentity() produced an error: %+v", err)
 	}
@@ -218,7 +191,7 @@ func TestStore_GetIdentities(t *testing.T) {
 	}
 
 	//get three
-	idu, err = s.GetIdentities(3, prng, 15)
+	err = s.ForEach(3, prng, 15, o)
 	if err != nil {
 		t.Errorf("GetIdentity() produced an error: %+v", err)
 	}
@@ -235,7 +208,7 @@ func TestStore_GetIdentities(t *testing.T) {
 	}
 
 	//get ten
-	idu, err = s.GetIdentities(10, prng, 15)
+	err = s.ForEach(10, prng, 15, o)
 	if err != nil {
 		t.Errorf("GetIdentity() produced an error: %+v", err)
 	}
@@ -252,7 +225,7 @@ func TestStore_GetIdentities(t *testing.T) {
 	}
 
 	//get fifty
-	idu, err = s.GetIdentities(50, prng, 15)
+	err = s.ForEach(50, prng, 15, o)
 	if err != nil {
 		t.Errorf("GetIdentity() produced an error: %+v", err)
 	}
@@ -269,7 +242,7 @@ func TestStore_GetIdentities(t *testing.T) {
 	}
 
 	//get 100
-	idu, err = s.GetIdentities(100, prng, 15)
+	err = s.ForEach(100, prng, 15, o)
 	if err != nil {
 		t.Errorf("GetIdentity() produced an error: %+v", err)
 	}
@@ -286,7 +259,7 @@ func TestStore_GetIdentities(t *testing.T) {
 	}
 
 	//get 1000, should only return 100
-	idu, err = s.GetIdentities(1000, prng, 15)
+	err = s.ForEach(1000, prng, 15, o)
 	if err != nil {
 		t.Errorf("GetIdentity() produced an error: %+v", err)
 	}
@@ -304,7 +277,13 @@ func TestStore_GetIdentities(t *testing.T) {
 
 	// get 100 a second time and make sure the order is not the same as a
 	// smoke test that the shuffle is working
-	idu2, err := s.GetIdentities(1000, prng, 15)
+	var idu2 []IdentityUse
+	o2 := func(a []IdentityUse) error {
+		idu2 = a
+		return nil
+	}
+
+	err = s.ForEach(1000, prng, 15, o2)
 	if err != nil {
 		t.Errorf("GetIdentity() produced an error: %+v", err)
 	}
@@ -329,7 +308,13 @@ func TestStore_GetIdentities_NoIdentities(t *testing.T) {
 	s := NewOrLoadStore(kv)
 	prng := rand.New(rand.NewSource(42))
 
-	idu, err := s.GetIdentities(5, prng, 15)
+	var idu []IdentityUse
+	o := func(a []IdentityUse) error {
+		idu = a
+		return nil
+	}
+
+	err := s.ForEach(5, prng, 15, o)
 	if err != nil {
 		t.Errorf("GetIdentities() produced an error: %+v", err)
 	}
@@ -350,27 +335,31 @@ func TestStore_GetIdentities_BadNum(t *testing.T) {
 	s := NewOrLoadStore(kv)
 	prng := rand.New(rand.NewSource(42))
 
-	_, err := s.GetIdentities(0, prng, 15)
+	o := func(a []IdentityUse) error {
+		return nil
+	}
+
+	err := s.ForEach(0, prng, 15, o)
 	if err == nil {
 		t.Errorf("GetIdentities() shoud error with bad num value")
 	}
 
-	_, err = s.GetIdentities(-1, prng, 15)
+	err = s.ForEach(-1, prng, 15, o)
 	if err == nil {
 		t.Errorf("GetIdentities() shoud error with bad num value")
 	}
 
-	_, err = s.GetIdentities(-100, prng, 15)
+	err = s.ForEach(-100, prng, 15, o)
 	if err == nil {
 		t.Errorf("GetIdentities() shoud error with bad num value")
 	}
 
-	_, err = s.GetIdentities(-1000000, prng, 15)
+	err = s.ForEach(-1000000, prng, 15, o)
 	if err == nil {
 		t.Errorf("GetIdentities() shoud error with bad num value")
 	}
 
-	_, err = s.GetIdentities(math.MinInt64, prng, 15)
+	err = s.ForEach(math.MinInt64, prng, 15, o)
 	if err == nil {
 		t.Errorf("GetIdentities() shoud error with bad num value")
 	}

cmix/identity/tracker.go

@@ -49,8 +49,8 @@ type Tracker interface {
 	StartProcesses() stoppable.Stoppable
 	AddIdentity(id *id.ID, validUntil time.Time, persistent bool)
 	RemoveIdentity(id *id.ID)
-	GetEphemeralIdentity(rng io.Reader, addressSize uint8) receptionID.IdentityUse
-	GetEphemeralIdentities(n int, rng io.Reader, addressSize uint8) ([]receptionID.IdentityUse, error)
+	ForEach(n int, rng io.Reader, addressSize uint8,
+		operator func([]receptionID.IdentityUse) error) error
 	GetIdentity(get *id.ID) (TrackedID, error)
 }
 
@@ -143,20 +143,14 @@ func (t *manager) RemoveIdentity(id *id.ID) {
 	t.deleteIdentity <- id
 }
 
-// GetEphemeralIdentity returns an ephemeral Identity to poll the network with.
-// It will return a fake identity if none are available.
-func (t *manager) GetEphemeralIdentity(rng io.Reader,
-	addressSize uint8) receptionID.IdentityUse {
-	return t.ephemeral.GetIdentity(rng, addressSize)
-}
-
-// GetEphemeralIdentities returns a fisher-yates shuffled list of up to 'num'
-// ephemeral identities. It will return a fake identity if none are available
+// ForEach passes a fisher-yates shuffled list of up to 'num'
+// ephemeral identities into the operation function. It will pass a
+// fake identity if none are available
 // and less than 'num' if less than 'num' are available.
 // 'num' must be positive non-zero
-func (t *manager) GetEphemeralIdentities(n int, rng io.Reader, addressSize uint8) (
-	[]receptionID.IdentityUse, error) {
-	return t.ephemeral.GetIdentities(n, rng, addressSize)
+func (t *manager) ForEach(n int, rng io.Reader, addressSize uint8,
+	operator func([]receptionID.IdentityUse) error) error {
+	return t.ephemeral.ForEach(n, rng, addressSize, operator)
 }
 
 // GetIdentity returns a currently tracked identity