Merge branch 'XX-4575/emojiValidationRewrite' into 'release'

XX-4575 / emoji validation rewrite

See merge request !91

.gitlab-ci.yml

@@ -77,22 +77,6 @@ build-workers:
       - release/
     expire_in: 1 hour
 
-emoji-update:
-  stage: build
-  except:
-    - tags
-  only:
-    - release
-    - master
-  script:
-    - go mod vendor -v
-    - mkdir -p release
-    - go run -ldflags '-w -s' -trimpath ./emoji/... -o emojiSet.json -v 0
-    - cp emojiSet.json release/
-  artifacts:
-    paths:
-      - release/
-    expire_in: 1 hour
 
 tag:
   stage: build
@@ -116,11 +100,9 @@ combine-artifacts:
     - echo $PIPELINE_JOBS
     - BUILD_JOB_JSON=$(echo $PIPELINE_JOBS | jq '.[] | select(.name=="build")')
     - BUILD_WORKERS_JOB_JSON=$(echo $PIPELINE_JOBS | jq '.[] | select(.name=="build-workers")')
-    - EMOJI_UPDATE_JOB_JSON=$(echo $PIPELINE_JOBS | jq '.[] | select(.name=="emoji-update")')
 
     - BUILD_JOB_ID=$(echo $BUILD_JOB_JSON | jq -r '.["id"]')
     - BUILD_WORKERS_JOB_ID=$(echo $BUILD_WORKERS_JOB_JSON | jq -r '.["id"]')
-    - EMOJI_UPDATE_JOB_ID=$(echo $EMOJI_UPDATE_JOB_JSON | jq -r '.["id"]')
 
     - rm -rf release
     - mkdir -p release
@@ -133,8 +115,6 @@ combine-artifacts:
     - 'curl --fail --location --header "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" --output release/channelsIndexedDbWorker.js $CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/jobs/$BUILD_WORKERS_JOB_ID/artifacts/release/channelsIndexedDbWorker.js'
     - 'curl --fail --location --header "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" --output release/dmIndexedDbWorker.js $CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/jobs/$BUILD_WORKERS_JOB_ID/artifacts/release/dmIndexedDbWorker.js'
     - 'curl --fail --location --header "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" --output release/logFileWorker.js $CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/jobs/$BUILD_WORKERS_JOB_ID/artifacts/release/logFileWorker.js'
-    - 'if [[ $CI_COMMIT_BRANCH =~ ^(release|master)$ ]]; then curl --fail --location --header "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" --output release/emojiSet.json $CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/jobs/$EMOJI_UPDATE_JOB_ID/artifacts/release/emojiSet.json; fi'
-
     - ls release
   artifacts:
     paths:

Makefile

@@ -32,9 +32,6 @@ worker_binaries:
 	GOOS=js GOARCH=wasm go build -ldflags '-w -s' -trimpath -o xxdk-dmIndexedDkWorker.wasm ./indexedDb/impl/dm/...
 	GOOS=js GOARCH=wasm go build -ldflags '-w -s' -trimpath -o xxdk-logFileWorker.wasm ./logging/workerThread/...
 
-emojis:
-	go run -ldflags '-w -s' -trimpath ./emoji/... -o emojiSet.json
-
 binaries: binary worker_binaries
 
 wasm_tests:

emoji/emojiMart.go

-////////////////////////////////////////////////////////////////////////////////
-// Copyright © 2022 xx foundation                                             //
-//                                                                            //
-// Use of this source code is governed by a license that can be found in the  //
-// LICENSE file.                                                              //
-////////////////////////////////////////////////////////////////////////////////
-
-package main
-
-// emojiID represents the alias for an emoji in emoji-mart. For example, the
-// alias for the emoji 💯 would be "100". This adheres strictly to how Emoji
-// Mart categorizes their emojis within the categories section of their JSON
-// file.
-type emojiID string
-
-// codepoint represents the Unicode codepoint or codepoints for an emoji. They
-// are in lowercase and if there are multiple codepoints, they are seperated by
-// a dash ("-"). For example, the emoji 💯 would have the codepoint "1f4af".
-type codepoint string
-
-// emojiMartSet is a representation of the JSON file format containing the emoji
-// list in emoji-mart. It matches the object EmojiMartData:
-// https://github.com/missive/emoji-mart/blob/main/packages/emoji-mart-data/index.d.ts
-//
-// Doc: https://github.com/missive/emoji-mart/
-// JSON example: https://github.com/missive/emoji-mart/blob/main/packages/emoji-mart-data/sets/14/native.json
-type emojiMartSet struct {
-	Categories []category         `json:"categories"`
-	Emojis     map[emojiID]emoji  `json:"emojis"`
-	Aliases    map[string]emojiID `json:"aliases"`
-	Sheet      sheet              `json:"sheet"`
-}
-
-// category adheres to the categories field within the EmojiMartData Javascript
-// interface.
-type category struct {
-	ID     string    `json:"id"`
-	Emojis []emojiID `json:"emojis"`
-}
-
-// emoji adheres to the emojis field within the EmojiMartData Javascript
-// interface.
-type emoji struct {
-	ID        emojiID  `json:"id"`
-	Name      string   `json:"name"`
-	Keywords  []string `json:"keywords"`
-	Skins     []skin   `json:"skins"`
-	Version   float64  `json:"version"`
-	Emoticons []string `json:"emoticons,omitempty"`
-}
-
-// skin adheres to the skins field within the EmojiMartData Javascript interface.
-type skin struct {
-	Unified codepoint `json:"unified"`
-	Native  string    `json:"native"`
-	X       float64   `json:"x,omitempty"`
-	Y       float64   `json:"y,omitempty"`
-}
-
-// sheet adheres to the sheet field within the EmojiMartData Javascript
-// interface.
-type sheet struct {
-	Cols float64 `json:"cols"`
-	Rows float64 `json:"rows"`
-}

emoji/emojiMart_test.go

-////////////////////////////////////////////////////////////////////////////////
-// Copyright © 2022 xx foundation                                             //
-//                                                                            //
-// Use of this source code is governed by a license that can be found in the  //
-// LICENSE file                                                               //
-////////////////////////////////////////////////////////////////////////////////
-
-package main
-
-import (
-	_ "embed"
-	"encoding/json"
-	"github.com/nsf/jsondiff"
-	"reflect"
-	"testing"
-)
-
-//go:embed emojiMart.json
-var emojiMartJson []byte
-
-// Tests that marshaling the emojiMartData object and unmarshalling that JSON
-// data back into an object does not cause loss in data.
-func Test_emojiMartData_JSON_Marshal_Unmarshal(t *testing.T) {
-	exampleData := emojiMartSet{
-		Categories: []category{
-			{ID: "100", Emojis: []emojiID{"100"}},
-			{ID: "21"},
-			{ID: "20"},
-		},
-		Emojis: map[emojiID]emoji{
-			"100": {
-				ID:       "100",
-				Name:     "Hundred Points",
-				Keywords: []string{"hunna"},
-				Skins:    nil,
-				Version:  0,
-			},
-		},
-		Aliases: map[string]emojiID{
-			"lady_beetle": "ladybug",
-		},
-		Sheet: sheet{
-			Cols: 5,
-			Rows: 12,
-		},
-	}
-
-	marshaled, err := json.Marshal(&exampleData)
-	if err != nil {
-		t.Fatalf("Failed to marshal: %+v", err)
-	}
-
-	unmarshalData := emojiMartSet{}
-	err = json.Unmarshal(marshaled, &unmarshalData)
-	if err != nil {
-		t.Fatalf("Failed to unmarshal: %+v", err)
-	}
-
-	if reflect.DeepEqual(unmarshalData, marshaled) {
-		t.Fatalf("Failed to unmarshal example and maintain original data."+
-			"\nExpected: %+v"+
-			"\nReceived: %+v", exampleData, unmarshalData)
-	}
-}
-
-// Tests that the emoji-mart example JSON can be JSON unmarshalled and
-// marshalled and that the result is semantically identical to the original.
-func Test_emojiMartDataJSON_Example(t *testing.T) {
-	emojiMart := &emojiMartSet{}
-	err := json.Unmarshal(emojiMartJson, emojiMart)
-	if err != nil {
-		t.Fatalf("Failed to unamrshal: %+v", err)
-	}
-
-	marshalled, err := json.Marshal(emojiMart)
-	if err != nil {
-		t.Fatalf("Failed to marshal: %+v", err)
-	}
-
-	opts := jsondiff.DefaultConsoleOptions()
-	d, s := jsondiff.Compare(emojiMartJson, marshalled, &opts)
-	if d != jsondiff.FullMatch {
-		t.Errorf("Diff failed for marshalled JSON: %s\nDifferences: %s", d, s)
-	}
-}

emoji/emojiSet.go

-////////////////////////////////////////////////////////////////////////////////
-// Copyright © 2022 xx foundation                                             //
-//                                                                            //
-// Use of this source code is governed by a license that can be found in the  //
-// LICENSE file                                                               //
-////////////////////////////////////////////////////////////////////////////////
-
-package main
-
-import (
-	"encoding/json"
-	"github.com/forPelevin/gomoji"
-	"github.com/pkg/errors"
-	jww "github.com/spf13/jwalterweatherman"
-	cEmoji "gitlab.com/elixxir/client/v4/emoji"
-	"strings"
-)
-
-// Set contains the set of emoji's that the backend supports. This object will
-// be used to remove all unsupported emojis in the given emoji-mart JSON set.
-type Set struct {
-	// replacementMap contains a list of emoji code-points in the emoji-mart set
-	// that must be replaced to adhere to backend recognized code-points.
-	replacementMap map[codepoint]skin
-
-	// supportedEmojis contains a list of all Unicode codepoints for the emojis
-	// that are supported. This allows for quick lookup when comparing against
-	// the emoji-mart of emojis.
-	supportedEmojis map[codepoint]struct{}
-}
-
-// NewSet initialises a new Emoji Set with all supported backend emojis.
-func NewSet() *Set {
-	return &Set{
-		replacementMap: map[codepoint]skin{
-			"2764-fe0f": {
-				Unified: "2764", // Has codepoint "2764-fe0f" in front-end
-				Native:  "❤",
-			},
-		},
-		supportedEmojis: emojiListToMap(cEmoji.SupportedEmojis()),
-	}
-}
-
-// SanitizeEmojiMartSet removes all unsupported emojis from the emoji-mart set
-// JSON. It also replaces any mismatched codepoints (where the same Emoji has
-// two different codepoints).
-func (s *Set) SanitizeEmojiMartSet(frontendEmojiSetJson []byte) ([]byte, error) {
-
-	// Unmarshal the emoji-mart set JSON
-	var frontEndEmojiSet emojiMartSet
-	err := json.Unmarshal(frontendEmojiSetJson, &frontEndEmojiSet)
-	if err != nil {
-		return nil, errors.Errorf(
-			"failed to unmarshal emoji-mart set JSON: %+v", err)
-	}
-
-	jww.DEBUG.Printf(
-		"Finding incompatible emojis and replacing mismatched codepoints.")
-
-	// Find all incompatible emojis in the front end set
-	emojisToRemove := s.findIncompatibleEmojis(&frontEndEmojiSet)
-
-	jww.DEBUG.Printf("Removing incompatible emojis.")
-
-	// Remove all incompatible emojis from the set
-	removeIncompatibleEmojis(&frontEndEmojiSet, emojisToRemove)
-
-	jww.INFO.Printf("Removed %d incompatible codepoints.", len(emojisToRemove))
-
-	return json.Marshal(frontEndEmojiSet)
-}
-
-// findIncompatibleEmojis returns a list of emojis in the emojiMartSet that are
-// not supported by the Set. Also, any emojiMartSet emoji codepoints that are
-// incompatible and have replacements (as defined in Set) are replaced.
-func (s *Set) findIncompatibleEmojis(set *emojiMartSet) (emojisToRemove []emojiID) {
-	// Iterate over all emojis in the emojiMartSet.Emojis list
-	for id, Emoji := range set.Emojis {
-		var newSkins []skin
-		for _, Skin := range Emoji.Skins {
-			// Determine if the emoji's codepoint should be replaced or removed
-			replacement, replace := s.replace(Skin.Unified)
-			if replace {
-				jww.TRACE.Printf("Replaced codepoint %q with %v for emoji %q",
-					Skin.Unified, replacement, id)
-				newSkins = append(newSkins, replacement)
-			} else if !s.remove(Skin.Unified) {
-				newSkins = append(newSkins, Skin)
-			} else {
-				jww.TRACE.Printf("Removed codepoint %q from emoji %q",
-					Skin.Unified, id)
-			}
-		}
-
-		if len(newSkins) > 0 {
-			// Write to the set the possible edits (if emojis were replaced
-			// or removed)
-			Emoji.Skins = newSkins
-			set.Emojis[id] = Emoji
-		} else {
-			// If all skins have been removed, then mark the emoji for removal
-			emojisToRemove = append(emojisToRemove, id)
-			jww.DEBUG.Printf("All skins removed for emoji %q", id)
-		}
-	}
-
-	return emojisToRemove
-}
-
-// removeIncompatibleEmojis removes all the emojis in emojisToRemove from the
-// emojiMartSet set.
-func removeIncompatibleEmojis(set *emojiMartSet, emojisToRemove []emojiID) {
-	jww.DEBUG.Printf(
-		"Removing %d emojis: %s", len(emojisToRemove), emojisToRemove)
-
-	// Remove all incompatible emojis from the emojiMartSet.Emojis list
-	for _, char := range emojisToRemove {
-		jww.TRACE.Printf("Removing %q from emojiMartSet.Emojis", char)
-		delete(set.Emojis, char)
-	}
-
-	// Remove all incompatible emojis from the emojiMartSet.Categories list
-	for _, cat := range set.Categories {
-		// Iterate over the emoji list backwards to make removal of elements
-		// from the slice easier
-		for i := len(cat.Emojis) - 1; i >= 0; i-- {
-			for _, char := range emojisToRemove {
-				if cat.Emojis[i] == char {
-					cat.Emojis = append(cat.Emojis[:i], cat.Emojis[i+1:]...)
-					jww.TRACE.Printf(
-						"Removing %q from emojiMartSet.Categories", char)
-				}
-			}
-		}
-	}
-
-	// Remove all incompatible emojis from the emojiMartSet.Aliases list
-	for alias, id := range set.Aliases {
-		for _, removedId := range emojisToRemove {
-			if id == removedId {
-				delete(set.Aliases, alias)
-				jww.TRACE.Printf(
-					"Removing %q from emojiMartSet.Aliases", alias)
-			}
-		}
-	}
-}
-
-// replace returns whether the front end Unicode codepoint must be replaced.
-// It will return a boolean on whether this codepoint needs to be replaced
-// and what the codepoint must be replaced with.
-func (s *Set) replace(code codepoint) (replacement skin, replace bool) {
-	replacement, replace = s.replacementMap[code]
-	return replacement, replace
-}
-
-// remove returns true if the code point should be removed from the parent list.
-func (s *Set) remove(code codepoint) bool {
-	_, exists := s.supportedEmojis[code]
-	return !exists
-}
-
-// emojiListToMap constructs a map for simple lookup for gomoji.Emoji's
-// Unicode codepoint.
-func emojiListToMap(list []gomoji.Emoji) map[codepoint]struct{} {
-	emojiMap := make(map[codepoint]struct{}, len(list))
-	for _, e := range list {
-		emojiMap[backToFrontCodePoint(e.CodePoint)] = struct{}{}
-	}
-	return emojiMap
-}
-
-// backToFrontCodePoint converts Unicode codepoint format found in gomoji.Emoji
-// to the one found in the emoji-mart JSON. The specific conversion is making it
-// lowercase and replacing " " with "-".
-func backToFrontCodePoint(code string) codepoint {
-	return codepoint(strings.ToLower(strings.ReplaceAll(code, " ", "-")))
-}

emoji/emojiSet_test.go

-////////////////////////////////////////////////////////////////////////////////
-// Copyright © 2022 xx foundation                                             //
-//                                                                            //
-// Use of this source code is governed by a license that can be found in the  //
-// LICENSE file                                                               //
-////////////////////////////////////////////////////////////////////////////////
-
-package main
-
-import (
-	"encoding/json"
-	"reflect"
-	"testing"
-)
-
-// Benchmarks the entire code path for sanitizing front end's emoji file.
-// This includes loading the file, parsing the data and sanitizing.
-// NOTE: This does not include writing the file due to limitations of the
-// embed.FS interface.
-func BenchmarkSet_SanitizeFrontendEmojiList(b *testing.B) {
-	backendSet := NewSet()
-
-	for i := 0; i < b.N; i++ {
-
-		// Sanitize front end example
-		_, err := backendSet.SanitizeEmojiMartSet(emojiMartJson)
-		if err != nil {
-			b.Fatalf("Failed to Sanitize front end emojis: %+v", err)
-		}
-	}
-}
-
-// Comprehensive test of Set.SanitizeEmojiMartSet using the front end example
-// JSON file (emojiMart.json).
-func TestSet_SanitizeFrontEndEmojis_FrontEndExample(t *testing.T) {
-
-	backendSet := NewSet()
-
-	// Sanitize front end example
-	sanitizedSetJson, err := backendSet.SanitizeEmojiMartSet(emojiMartJson)
-	if err != nil {
-		t.Fatalf("Failed to Sanitize front end emojis: %+v", err)
-	}
-
-	// Unmarshal front end example
-	unsanitizedSet := &emojiMartSet{}
-	err = json.Unmarshal(emojiMartJson, unsanitizedSet)
-	if err != nil {
-		t.Fatalf("Failed to unmarshal unsanitized set: %+v", err)
-	}
-
-	// Unmarshal sanitization of front end example
-	sanitizedSet := &emojiMartSet{}
-	err = json.Unmarshal(sanitizedSetJson, sanitizedSet)
-	if err != nil {
-		t.Fatalf("Failed to unmarshal sanitized set: +%v", err)
-	}
-
-	// The front end example has known unsanitary emojis.
-	// Therefore, sanitization of the front end example
-	// should not contain the exact same data.
-	if reflect.DeepEqual(sanitizedSet, unsanitizedSet) {
-		t.Fatalf("No evidence of sanitization performed")
-	}
-
-	// Check of replacement of the heart emoji (❤️).
-	// This is the known unsanitary emoji (it is not supported
-	// by backend's library).
-	heart, exists := sanitizedSet.Emojis["heart"]
-	if !exists {
-		t.Fatalf("Heart emoji was removed when it should have been replaced")
-	}
-
-	expectedHeartReplacement := skin{
-		Unified: "2764",
-		Native:  "❤",
-	}
-
-	if !reflect.DeepEqual(expectedHeartReplacement, heart.Skins[0]) {
-		t.Fatalf("Heart emoji was not replaced as expected."+
-			"\nExpected: %+v"+
-			"\nReceived: %+v", expectedHeartReplacement, heart.Skins[0])
-	}
-}
-
-// Test of Set.findIncompatibleEmojis using the front end example
-// JSON file (emojiMart.json).
-func TestSet_findIncompatibleEmojis_FrontEndExample(t *testing.T) {
-	backendSet := NewSet()
-
-	// Unmarshal front end example
-	unsanitizedSet := &emojiMartSet{}
-	err := json.Unmarshal(emojiMartJson, unsanitizedSet)
-	if err != nil {
-		t.Fatalf("Failed to unmarshal unsanitized set: %+v", err)
-	}
-
-	emojisToRemove := backendSet.findIncompatibleEmojis(unsanitizedSet)
-	if len(emojisToRemove) != 0 {
-		t.Fatalf("Front end example should not contain any removable emojis.")
-	}
-}
-
-// Tests Set.findIncompatibleEmojis using a custom emojiMartSet object.
-// There is one ID  with skins that should be marked as removable as all skins
-// are invalid when compared against backend. There is another ID that should
-// not be removed, and should not be returned by set.findIncompatibleEmojis.
-func TestSet_findIncompatibleEmojis_RemovableExample(t *testing.T) {
-	backendSet := NewSet()
-
-	// Construct custom emojiMartSet.
-	toBeRemovedId := emojiID("shouldBeRemoved")
-	emd := constructRemovableEmojiSet(toBeRemovedId)
-
-	// Construct a removable list
-	removedEmojis := backendSet.findIncompatibleEmojis(emd)
-
-	// Ensure only oen emoji is removed
-	if len(removedEmojis) != 1 {
-		t.Fatalf("findIncompatibleEmojis should only mark one emoji for removal.")
-	}
-
-	// Ensure the single removed emoji is the expected emoji
-	if removedEmojis[0] != toBeRemovedId {
-		t.Fatalf("findIncompatibleEmojis should have found %s to be removable",
-			toBeRemovedId)
-	}
-}
-
-// Tests that Set.removeIncompatibleEmojis will modify the passed in
-// emojiMartSet according to the list of emojis to remove.
-func TestSet_removeIncompatibleEmojis(t *testing.T) {
-	backendSet := NewSet()
-
-	// Construct custom emojiMartSet.
-	toBeRemovedId := emojiID("shouldBeRemoved")
-	emd := constructRemovableEmojiSet(toBeRemovedId)
-
-	// Construct a removable list
-	removedEmojis := backendSet.findIncompatibleEmojis(emd)
-
-	removeIncompatibleEmojis(emd, removedEmojis)
-
-	// Test that categories has been modified
-	for _, cat := range emd.Categories {
-		for _, e := range cat.Emojis {
-			if e == toBeRemovedId {
-				t.Fatalf("EmojiID %q was never removed from "+
-					"emojiMartSet.Categories", toBeRemovedId)
-			}
-		}
-	}
-
-	// Test that the emoji map has been modified
-	if _, exists := emd.Emojis[toBeRemovedId]; exists {
-		t.Fatalf("EmojiId %s was not removed from emojiMartSet.Emojis",
-			toBeRemovedId)
-	}
-
-	// Test that the alias list has been modified
-	for _, id := range emd.Aliases {
-		if id == toBeRemovedId {
-			t.Fatalf("EmojiId %s twas not removed from emojiMartSet.Aliases",
-				toBeRemovedId)
-		}
-	}
-}
-
-// Tests backToFrontCodePoint converts backend Unicode codepoints to their
-// front end equivalent.
-func Test_backToFrontCodePoint(t *testing.T) {
-	// Input for backend codepoints and their front end formatted pairings
-	tests := []struct {
-		input  string
-		output codepoint
-	}{
-		{"0023 FE0F 20E3", "0023-fe0f-20e3"}, // Single-rune emoji (\u1F44B)
-		{"002A FE0F 20E3", "002a-fe0f-20e3"}, // Duel-rune emoji with race modification (\u1F44B\u1F3FF)
-		{"00A9 FE0F", "00a9-fe0f"},
-		{"1F481 1F3FC 200D 2640 FE0F", "1f481-1f3fc-200d-2640-fe0f"},
-		{"1F481 1F3FE 200D 2642 FE0F", "1f481-1f3fe-200d-2642-fe0f"},
-		{"1F481 1F3FF", "1f481-1f3ff"},
-		{"1F481 1F3FF 200D 2642 FE0F", "1f481-1f3ff-200d-2642-fe0f"},
-		{"1F469 1F3FB 200D 1F9B0", "1f469-1f3fb-200d-1f9b0"},
-		{"1F378", "1f378"},
-		{"1F377", "1f377"},
-		{"1F376", "1f376"},
-	}
-
-	// Test that for all examples, all backend codepoints are converted
-	// to front end codepoints
-	for _, test := range tests {
-		received := backToFrontCodePoint(test.input)
-		if received != test.output {
-			t.Fatalf("Incorrect codepoint for %q.\nexpected: %s\nreceived: %s",
-				test.input, test.output, received)
-		}
-	}
-
-}
-
-// constructRemovableEmojiSet returns an emojiMartSet object used for testing.
-// This object will contain data that should be marked as removable by
-// Set.findIncompatibleEmojis. This removable data from
-// Set.findIncompatibleEmojis can be passed to removeIncompatibleEmojis.
-func constructRemovableEmojiSet(toBeRemovedId emojiID) *emojiMartSet {
-	return &emojiMartSet{
-		Categories: []category{
-			{Emojis: []emojiID{
-				toBeRemovedId,
-				"1",
-				"two",
-				"tree",
-				"for",
-				"golden rings",
-			}},
-		},
-		Emojis: map[emojiID]emoji{
-			toBeRemovedId: {
-				Skins: []skin{{
-					Unified: "00A9 FE0F 20F3",
-					Native:  "",
-				}, {
-					Unified: "AAAA FE0F 20F3",
-					Native:  "",
-				}},
-			},
-			"shouldNotBeRemoved": {
-				Skins: []skin{{
-					Unified: "1f9e1",
-					Native:  "🧡",
-				}},
-			},
-		},
-		Aliases: map[string]emojiID{
-			"test":   toBeRemovedId,
-			"tester": "safe",
-			"alias":  "will_not",
-			"secret": "beRemoved",
-		},
-	}
-}

emoji/main.go

-////////////////////////////////////////////////////////////////////////////////
-// Copyright © 2022 xx foundation                                             //
-//                                                                            //
-// Use of this source code is governed by a license that can be found in the  //
-// LICENSE file.                                                              //
-////////////////////////////////////////////////////////////////////////////////
-
-// package main is its own utility that is compiled separate from xxdk-WASM. It
-// is used only to produce a compatible emoji file to be used by the frontend
-// and is not a WASM module itself.
-
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"github.com/spf13/cobra"
-	jww "github.com/spf13/jwalterweatherman"
-	"gitlab.com/xx_network/primitives/utils"
-	"io"
-	"log"
-	"net/http"
-	"os"
-	"strconv"
-)
-
-// emojiMartUrl is the URL pointing to the native.JSON from emoji-mart that is
-// used by front end.
-//
-// NOTE: This points specifically to set version 14 of the emoji-mart data. This
-// URL should be updated if new sets become available.
-const emojiMartUrl = "https://raw.githubusercontent.com/missive/emoji-mart/main/packages/emoji-mart-data/sets/14/native.json"
-
-// Flag variables.
-var (
-	requestURL, outputPath, logFile string
-	logLevel                        int
-)
-
-func main() {
-	if err := cmd.Execute(); err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-}
-
-// Downloads the emoji file (from emoji-mart) and sanitizes that
-// list. Sanitization removes all emojis not supported by the backend. The
-// sanitized JSON is returned via a file specified by the user. Refer to the
-// flags for details.
-var cmd = &cobra.Command{
-	Use: "sanitizeEmojis",
-	Short: "Downloads the emoji file (from emoji-mart) and sanitizes that " +
-		"list. Sanitization removes all emojis not supported by the backend. " +
-		"The sanitized JSON is returned via a file specified by the user." +
-		"Refer to the flags for details.",
-	Args: cobra.NoArgs,
-	Run: func(cmd *cobra.Command, args []string) {
-
-		// Initialize the logging
-		initLog(jww.Threshold(logLevel), logFile)
-
-		// Retrieve emoji-mart file from URL
-		jww.INFO.Printf("Requesting file %s", requestURL)
-		resp, err := http.Get(requestURL)
-		if err != nil {
-			jww.FATAL.Panicf(
-				"Failed to retrieve emoji-mart JSON from URL: %+v", err)
-		} else if resp.StatusCode != http.StatusOK {
-			jww.FATAL.Panicf("Bad status: %s", resp.Status)
-		}
-
-		jww.DEBUG.Printf("Received HTTP response: %+v", resp)
-
-		// Read HTTP response into byte slice
-		var buf bytes.Buffer
-		_, err = buf.ReadFrom(resp.Body)
-		if err != nil {
-			jww.FATAL.Panicf("Failed to read from HTTP response: %+v", err)
-		}
-		if err = resp.Body.Close(); err != nil {
-			jww.FATAL.Panicf("Failed to close HTTP response: %+v", err)
-		}
-		emojiMartJson := buf.Bytes()
-
-		jww.DEBUG.Printf("Read %d bytes of JSON file", len(emojiMartJson))
-
-		// Sanitize the JSON file
-		backendSet := NewSet()
-		sanitizedJSON, err := backendSet.SanitizeEmojiMartSet(emojiMartJson)
-		if err != nil {
-			jww.FATAL.Panicf("Failed to sanitize emoji-mart list: %+v", err)
-		}
-
-		jww.DEBUG.Printf("Sanitised JSON file.")
-
-		// Write sanitized JSON to file
-		err = utils.WriteFileDef(outputPath, sanitizedJSON)
-		if err != nil {
-			jww.FATAL.Panicf(
-				"Failed to write sanitized emojis to filepath %s: %+v",
-				outputPath, err)
-		}
-
-		jww.INFO.Printf("Wrote sanitised JSON file to %s", outputPath)
-	},
-}
-
-// init is the initialization function for Cobra which defines flags.
-func init() {
-	cmd.Flags().StringVarP(&requestURL, "url", "u", emojiMartUrl,
-		"URL to download emoji-mart JSON file.")
-	cmd.Flags().StringVarP(&outputPath, "output", "o", "output.json",
-		"Output JSON file path.")
-	cmd.Flags().StringVarP(&logFile, "log", "l", "-",
-		"Log output path. By default, logs are printed to stdout. "+
-			"To disable logging, set this to empty (\"\").")
-	cmd.Flags().IntVarP(&logLevel, "logLevel", "v", 4,
-		"Verbosity level of logging. 0 = TRACE, 1 = DEBUG, 2 = INFO, "+
-			"3 = WARN, 4 = ERROR, 5 = CRITICAL, 6 = FATAL")
-}
-
-// initLog will enable JWW logging to the given log path with the given
-// threshold. If log path is empty, then logging is not enabled. Panics if the
-// log file cannot be opened or if the threshold is invalid.
-func initLog(threshold jww.Threshold, logPath string) {
-	if logPath == "" {
-		// Do not enable logging if no log file is set
-		return
-	} else if logPath != "-" {
-		// Set the log file if stdout is not selected
-
-		// Disable stdout output
-		jww.SetStdoutOutput(io.Discard)
-
-		// Use log file
-		logOutput, err :=
-			os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
-		if err != nil {
-			panic(err)
-		}
-		jww.SetLogOutput(logOutput)
-	}
-
-	if threshold < jww.LevelTrace || threshold > jww.LevelFatal {
-		panic("Invalid log threshold: " + strconv.Itoa(int(threshold)))
-	}
-
-	// Display microseconds if the threshold is set to TRACE or DEBUG
-	if threshold == jww.LevelTrace || threshold == jww.LevelDebug {
-		jww.SetFlags(log.LstdFlags | log.Lmicroseconds)
-	}
-
-	// Enable logging
-	jww.SetStdoutThreshold(threshold)
-	jww.SetLogThreshold(threshold)
-	jww.INFO.Printf("Log level set to: %s", threshold)
-}

go.mod

@@ -4,13 +4,10 @@ go 1.19
 
 require (
 	github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2
-	github.com/forPelevin/gomoji v1.1.8
 	github.com/hack-pad/go-indexeddb v0.2.0
-	github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249
 	github.com/pkg/errors v0.9.1
-	github.com/spf13/cobra v1.5.0
 	github.com/spf13/jwalterweatherman v1.1.0
-	gitlab.com/elixxir/client/v4 v4.6.2-0.20230403163818-19dafe415a56
+	gitlab.com/elixxir/client/v4 v4.6.2-0.20230403172317-1b360bb9aa50
 	gitlab.com/elixxir/crypto v0.0.7-0.20230322175717-4a3b5a24bdf4
 	gitlab.com/elixxir/primitives v0.0.3-0.20230214180039-9a25e2d3969c
 	gitlab.com/xx_network/crypto v0.0.5-0.20230214003943-8a09396e95dd
@@ -27,40 +24,26 @@ require (
 	github.com/cloudflare/circl v1.2.0 // indirect
 	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
 	github.com/elliotchance/orderedmap v1.4.0 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/gobwas/ws v1.1.0 // indirect
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/improbable-eng/grpc-web v0.15.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.15.9 // indirect
 	github.com/klauspost/cpuid/v2 v2.1.0 // indirect
-	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mattn/go-sqlite3 v1.14.15 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
 	github.com/oasisprotocol/curve25519-voi v0.0.0-20221003100820-41fad3beba17 // indirect
 	github.com/oasisprotocol/deoxysii v0.0.0-20220228165953-2091330c22b7 // indirect
-	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.2 // indirect
-	github.com/pkg/profile v1.6.0 // indirect
-	github.com/rivo/uniseg v0.4.3 // indirect
 	github.com/rs/cors v1.8.2 // indirect
 	github.com/sethvargo/go-diceware v0.3.0 // indirect
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
 	github.com/soheilhy/cmux v0.1.5 // indirect
-	github.com/spf13/afero v1.9.2 // indirect
-	github.com/spf13/cast v1.5.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.12.0 // indirect
-	github.com/subosito/gotenv v1.4.0 // indirect
 	github.com/ttacon/builder v0.0.0-20170518171403-c099f663e1c2 // indirect
 	github.com/ttacon/libphonenumber v1.2.1 // indirect
 	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
@@ -81,9 +64,6 @@ require (
 	google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
 	google.golang.org/grpc v1.49.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
-	gopkg.in/ini.v1 v1.66.6 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gorm.io/driver/sqlite v1.4.4 // indirect
 	gorm.io/gorm v1.24.3 // indirect
 	nhooyr.io/websocket v1.8.7 // indirect

main.go

@@ -118,6 +118,7 @@ func main() {
 
 	// wasm/emoji.go
 	js.Global().Set("SupportedEmojis", js.FuncOf(wasm.SupportedEmojis))
+	js.Global().Set("SupportedEmojisMap", js.FuncOf(wasm.SupportedEmojisMap))
 	js.Global().Set("ValidateReaction", js.FuncOf(wasm.ValidateReaction))
 
 	// wasm/errors.go

wasm/emoji.go

@@ -10,49 +10,96 @@
 package wasm
 
 import (
-	"encoding/json"
+	"syscall/js"
+
 	"gitlab.com/elixxir/client/v4/bindings"
-	"gitlab.com/elixxir/client/v4/emoji"
 	"gitlab.com/elixxir/xxdk-wasm/utils"
-	"syscall/js"
 )
 
 // SupportedEmojis returns a list of emojis that are supported by the backend.
+// The list includes all emojis described in [UTS #51 section A.1: Data Files].
 //
 // Returns:
-//   - JSON of an array of gomoji.Emoji (Uint8Array).
+//   - JSON of an array of emoji.Emoji (Uint8Array).
 //   - Throws a TypeError if marshalling the JSON fails.
 //
 // Example JSON:
 //
-//	[
-//	  {
-//	    "slug": "smiling-face",
-//	    "character": "☺️",
-//	    "unicode_name": "E0.6 smiling face",
-//	    "code_point": "263A FE0F",
-//	    "group": "Smileys \u0026 Emotion",
-//	    "sub_group": "face-affection"
-//	  },
-//	  {
-//	    "slug": "frowning-face",
-//	    "character": "☹️",
-//	    "unicode_name": "E0.7 frowning face",
-//	    "code_point": "2639 FE0F",
-//	    "group": "Smileys \u0026 Emotion",
-//	    "sub_group": "face-concerned"
-//	  },
-//	  {
-//	    "slug": "banana",
-//	    "character": "�",
-//	    "unicode_name": "E0.6 banana",
-//	    "code_point": "1F34C",
-//	    "group": "Food \u0026 Drink",
-//	    "sub_group": "food-fruit"
-//	  }
-//	]
+//		[
+//		  {
+//	     "character": "☹️",
+//	     "name": "frowning face",
+//	     "comment": "E0.7",
+//	     "codePoint": "2639 FE0F",
+//	     "group": "Smileys \u0026 Emotion",
+//	     "subgroup": "face-concerned"
+//		  },
+//		  {
+//	     "character": "☺️",
+//	     "name": "smiling face",
+//	     "comment": "E0.6",
+//	     "codePoint": "263A FE0F",
+//	     "group": "Smileys \u0026 Emotion",
+//	     "subgroup": "face-affection"
+//		  },
+//		  {
+//	     "character": "☢️",
+//	     "name": "radioactive",
+//	     "comment": "E1.0",
+//	     "codePoint": "2622 FE0F",
+//	     "group": "Symbols",
+//	     "subgroup": "warning"
+//		  }
+//		]
+//
+// [UTS #51 section A.1: Data Files]: https://www.unicode.org/reports/tr51/#Data_Files
 func SupportedEmojis(js.Value, []js.Value) any {
-	data, err := json.Marshal(emoji.SupportedEmojis())
+	data, err := bindings.SupportedEmojis()
+	if err != nil {
+		utils.Throw(utils.TypeError, err)
+		return nil
+	}
+
+	return utils.CopyBytesToJS(data)
+}
+
+// SupportedEmojisMap returns a map of emojis that are supported by the backend
+// as described by [SupportedEmojis].
+//
+// Returns:
+//   - JSON of a map of emoji.Emoji (Uint8Array).
+//   - Throws a TypeError if marshalling the JSON fails.
+//
+// Example JSON:
+//
+//		[
+//		  {
+//	     "character": "☹️",
+//	     "name": "frowning face",
+//	     "comment": "E0.7",
+//	     "codePoint": "2639 FE0F",
+//	     "group": "Smileys \u0026 Emotion",
+//	     "subgroup": "face-concerned"
+//		  },
+//		  {
+//	     "character": "☺️",
+//	     "name": "smiling face",
+//	     "comment": "E0.6",
+//	     "codePoint": "263A FE0F",
+//	     "group": "Smileys \u0026 Emotion",
+//	     "subgroup": "face-affection"
+//		  },
+//		  {
+//	     "character": "☢️",
+//	     "name": "radioactive",
+//	     "comment": "E1.0",
+//	     "codePoint": "2622 FE0F",
+//	     "group": "Symbols",
+//	     "subgroup": "warning"
+//		  }
+//		]
+func SupportedEmojisMap(js.Value, []js.Value) any {
+	data, err := bindings.SupportedEmojisMap()
 	if err != nil {
 		utils.Throw(utils.TypeError, err)
 		return nil
@@ -61,10 +108,11 @@ func SupportedEmojis(js.Value, []js.Value) any {
 	return utils.CopyBytesToJS(data)
 }
 
-// ValidateReaction checks that the reaction only contains a single emoji.
+// ValidateReaction checks that the reaction only contains a single grapheme
+// (one or more codepoints that appear as a single character to the user).
 //
 // Parameters:
-//   - args[0] - The reaction emoji to validate (string).
+//   - args[0] - The reaction to validate (string).
 //
 // Returns:
 //   - If the reaction is valid, returns null.