Fix flags and add more logging

.gitlab-ci.yml

@@ -51,7 +51,7 @@ emoji-update:
   script:
     - go mod vendor -v
     - mkdir -p release
-    - go run -ldflags '-w -s' ./emoji/... -o emojiSet.json -l - -v 2
+    - go run -ldflags '-w -s' ./emoji/... -o emojiSet.json -v 2
     - cp emojiSet.json release/
   artifacts:
     paths:

emoji/emojiSet.go

@@ -55,16 +55,19 @@ func (s *Set) SanitizeEmojiMartSet(frontendEmojiSetJson []byte) ([]byte, error)
 			"failed to unmarshal emoji-mart set JSON: %+v", err)
 	}
 
-	jww.INFO.Printf("[SanitizeEmojiMartSet] Finding incompatible emojis...")
+	jww.INFO.Printf(
+		"Finding incompatible emojis and replacing mismatched codepoints.")
 
 	// Find all incompatible emojis in the front end set
 	emojisToRemove := s.findIncompatibleEmojis(&frontEndEmojiSet)
 
-	jww.INFO.Printf("[SanitizeEmojiMartSet] Removing incompatible emojis...")
+	jww.INFO.Printf("Removing incompatible emojis.")
 
 	// Remove all incompatible emojis from the set
 	removeIncompatibleEmojis(&frontEndEmojiSet, emojisToRemove)
 
+	jww.INFO.Printf("Removed %d incompatible codepoints.", len(emojisToRemove))
+
 	return json.Marshal(frontEndEmojiSet)
 }
 
@@ -79,9 +82,14 @@ func (s *Set) findIncompatibleEmojis(set *emojiMartSet) (emojisToRemove []emojiI
 			// Determine if the emoji's codepoint should be replaced or removed
 			replacement, replace := s.replace(Skin.Unified)
 			if replace {
+				jww.TRACE.Printf("Replaced codepoint %q with %q for emoji %q",
+					Skin.Unified, replacement, id)
 				newSkins = append(newSkins, replacement)
 			} else if !s.remove(Skin.Unified) {
 				newSkins = append(newSkins, Skin)
+			} else {
+				jww.TRACE.Printf("Removed codepoint %q from emoji %q",
+					Skin.Unified, id)
 			}
 		}
 
@@ -93,6 +101,7 @@ func (s *Set) findIncompatibleEmojis(set *emojiMartSet) (emojisToRemove []emojiI
 		} else {
 			// If all skins have been removed, then mark the emoji for removal
 			emojisToRemove = append(emojisToRemove, id)
+			jww.DEBUG.Printf("All skins removed for emoji %q", id)
 		}
 	}
 
@@ -102,8 +111,12 @@ func (s *Set) findIncompatibleEmojis(set *emojiMartSet) (emojisToRemove []emojiI
 // removeIncompatibleEmojis removes all the emojis in emojisToRemove from the
 // emojiMartSet set.
 func removeIncompatibleEmojis(set *emojiMartSet, emojisToRemove []emojiID) {
+	jww.DEBUG.Printf(
+		"Removing %d emojis: %s", len(emojisToRemove), emojisToRemove)
+
 	// Remove all incompatible emojis from the emojiMartSet.Emojis list
 	for _, char := range emojisToRemove {
+		jww.TRACE.Printf("Removing %q from emojiMartSet.Emojis", char)
 		delete(set.Emojis, char)
 	}
 
@@ -115,6 +128,8 @@ func removeIncompatibleEmojis(set *emojiMartSet, emojisToRemove []emojiID) {
 			for _, char := range emojisToRemove {
 				if cat.Emojis[i] == char {
 					cat.Emojis = append(cat.Emojis[:i], cat.Emojis[i+1:]...)
+					jww.TRACE.Printf(
+						"Removing %q from emojiMartSet.Categories", char)
 				}
 			}
 		}
@@ -125,6 +140,8 @@ func removeIncompatibleEmojis(set *emojiMartSet, emojisToRemove []emojiID) {
 		for _, removedId := range emojisToRemove {
 			if id == removedId {
 				delete(set.Aliases, alias)
+				jww.TRACE.Printf(
+					"Removing %q from emojiMartSet.Aliases", alias)
 			}
 		}
 	}

emoji/main.go

@@ -12,8 +12,9 @@ import (
 	"fmt"
 	"github.com/spf13/cobra"
 	jww "github.com/spf13/jwalterweatherman"
-	"github.com/spf13/viper"
 	"gitlab.com/xx_network/primitives/utils"
+	"io"
+	"log"
 	"net/http"
 	"os"
 )
@@ -25,25 +26,24 @@ import (
 // URL should be updated if new sets become available.
 const emojiMartUrl = "https://raw.githubusercontent.com/missive/emoji-mart/main/packages/emoji-mart-data/sets/14/native.json"
 
-// Flag constants.
-const (
-	sanitizedOutputFlag = "output"
-	logLevelFlag        = "logLevel"
-	logFileFlag         = "logFile"
+// Flag variables.
+var (
+	requestURL, outputPath, logFile string
+	logLevel                        int
 )
 
 func main() {
-	if err := sanitizeEmojis.Execute(); err != nil {
+	if err := cmd.Execute(); err != nil {
 		fmt.Println(err)
 		os.Exit(1)
 	}
 }
 
-// sanitizeEmojis Downloads the emoji file (from emoji-mart) and sanitizes that
+// Downloads the emoji file (from emoji-mart) and sanitizes that
 // list. Sanitization removes all emojis not supported by the backend. The
-// sanitized JSON is returned via a file specified by the user. Refer to the flags
-// for details.
-var sanitizeEmojis = &cobra.Command{
+// sanitized JSON is returned via a file specified by the user. Refer to the
+// flags for details.
+var cmd = &cobra.Command{
 	Use: "sanitizeEmojis",
 	Short: "Downloads the emoji file (from emoji-mart) and sanitizes that " +
 		"list. Sanitization removes all emojis not supported by the backend. " +
@@ -53,20 +53,22 @@ var sanitizeEmojis = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 
 		// Initialize the logging if set
-		if logFile := viper.GetString(logFileFlag); logFile != "" {
-			initLog(viper.GetInt(logFileFlag), logFile)
+		if logFile != "" {
+			initLog(logLevel, logFile)
 		}
 
-		jww.INFO.Printf("Retrieving emoji-mart JSON file...")
-
 		// Retrieve emoji-mart file from URL
-		resp, err := http.Get(emojiMartUrl)
+		jww.INFO.Printf("Requesting file %s", requestURL)
+		resp, err := http.Get(requestURL)
 		if err != nil {
 			jww.FATAL.Panicf(
 				"Failed to retrieve emoji-mart JSON from URL: %+v", err)
+		} else if resp.StatusCode != http.StatusOK {
+			jww.FATAL.Panicf("Bad status: %s", resp.Status)
 		}
 
-		jww.INFO.Printf("Reading emoji-mart JSON file into bytes...")
+		jww.INFO.Printf("Received HTTP response: %s", resp.Status)
+		jww.DEBUG.Printf("Response: %+v", resp)
 
 		// Read HTTP response into byte slice
 		var buf bytes.Buffer
@@ -79,7 +81,7 @@ var sanitizeEmojis = &cobra.Command{
 		}
 		emojiMartJson := buf.Bytes()
 
-		jww.INFO.Printf("Sanitizing emoji-mart JSON...")
+		jww.INFO.Printf("Read %d bytes of JSON file", len(emojiMartJson))
 
 		// Sanitize the JSON file
 		backendSet := NewSet()
@@ -88,52 +90,61 @@ var sanitizeEmojis = &cobra.Command{
 			jww.FATAL.Panicf("Failed to sanitize emoji-mart list: %+v", err)
 		}
 
-		jww.INFO.Printf("Outputting sanitized emoji JSON to file...")
+		jww.INFO.Printf("Sanitised JSON file.")
 
 		// Write sanitized JSON to file
-		sanitizedOutputFilePath := viper.GetString(sanitizedOutputFlag)
-		err = utils.WriteFileDef(sanitizedOutputFilePath, sanitizedJSON)
+		err = utils.WriteFileDef(outputPath, sanitizedJSON)
 		if err != nil {
 			jww.FATAL.Panicf(
 				"Failed to write sanitized emojis to filepath %s: %+v",
-				sanitizedOutputFilePath, err)
+				outputPath, err)
 		}
+
+		jww.INFO.Printf("Wrote sanitised JSON file to %s", outputPath)
 	},
 }
 
-// init is the initialization function for Cobra which defines commands
-// and flags.
+// init is the initialization function for Cobra which defines flags.
 func init() {
-	// Here you will define your flags and configuration settings.
-	// Cobra supports persistent flags, which, if defined here,
-	// will be global for your application.
-	sanitizeEmojis.PersistentFlags().StringP(sanitizedOutputFlag, "o",
-		"output.json",
-		"File path that the sanitized JSON file will be outputted to.")
-	err := viper.BindPFlag(sanitizedOutputFlag, sanitizeEmojis.PersistentFlags().
-		Lookup(sanitizedOutputFlag))
-	if err != nil {
-		jww.FATAL.Panicf(
-			"Failed to bind pf flag to %q: %+v", sanitizedOutputFlag, err)
+	cmd.Flags().StringVarP(&requestURL, "url", "u", emojiMartUrl,
+		"URL to download emoji-mart JSON file.")
+	cmd.Flags().StringVarP(&outputPath, "output", "o", "output.json",
+		"Output JSON file path.")
+	cmd.Flags().StringVarP(&logFile, "log", "l", "-",
+		"Log output path. By default, logs are printed to stdout. "+
+			"To disable logging, set this to empty.")
+	cmd.Flags().IntVarP(&logLevel, "logLevel", "v", 0,
+		"Verbosity level of logging. 0 = INFO, 1 = DEBUG, 2 = TRACE")
 }
 
-	sanitizeEmojis.PersistentFlags().StringP(logFileFlag, "l", "",
-		"Path to the log output path. By default, this flag is not set "+
-			"so a log will not be created unless specified.")
-	err = viper.BindPFlag(logFileFlag, sanitizeEmojis.PersistentFlags().
-		Lookup(logFileFlag))
-	if err != nil {
-		jww.FATAL.Panicf(
-			"Failed to bind pf flag to %q: %+v", logFileFlag, err)
-	}
+// initLog will enable JWW logging.
+func initLog(threshold int, logPath string) {
+	if logPath != "-" && logPath != "" {
+		// Disable stdout output
+		jww.SetStdoutOutput(io.Discard)
 
-	sanitizeEmojis.PersistentFlags().IntP(logLevelFlag, "v", 0,
-		"Verbosity level of logging. This defaults to 0. ")
-	err = viper.BindPFlag(logLevelFlag, sanitizeEmojis.PersistentFlags().
-		Lookup(logLevelFlag))
+		// Use log file
+		logOutput, err :=
+			os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
 		if err != nil {
-		jww.FATAL.Panicf(
-			"Failed to bind pf flag to %q: %+v", logLevelFlag, err)
+			panic(err)
+		}
+		jww.SetLogOutput(logOutput)
 	}
 
+	if threshold > 1 {
+		jww.SetStdoutThreshold(jww.LevelTrace)
+		jww.SetLogThreshold(jww.LevelTrace)
+		jww.SetFlags(log.LstdFlags | log.Lmicroseconds)
+		jww.INFO.Printf("log level set to: %s", jww.LevelTrace)
+	} else if threshold == 1 {
+		jww.SetStdoutThreshold(jww.LevelDebug)
+		jww.SetLogThreshold(jww.LevelDebug)
+		jww.SetFlags(log.LstdFlags | log.Lmicroseconds)
+		jww.INFO.Printf("log level set to: %s", jww.LevelDebug)
+	} else {
+		jww.SetStdoutThreshold(jww.LevelInfo)
+		jww.SetLogThreshold(jww.LevelInfo)
+		jww.INFO.Printf("log level set to: %s", jww.LevelInfo)
+	}
 }

emoji/utils.go

-////////////////////////////////////////////////////////////////////////////////
-// Copyright © 2022 xx foundation                                             //
-//                                                                            //
-// Use of this source code is governed by a license that can be found in the  //
-// LICENSE file.                                                              //
-////////////////////////////////////////////////////////////////////////////////
-
-package main
-
-import (
-	jww "github.com/spf13/jwalterweatherman"
-	"io"
-	"log"
-	"os"
-)
-
-// initLog will enable JWW logging.
-func initLog(threshold int, logPath string) {
-	if logPath != "-" && logPath != "" {
-		// Disable stdout output
-		jww.SetStdoutOutput(io.Discard)
-		// Use log file
-		logOutput, err :=
-			os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
-		if err != nil {
-			panic(err)
-		}
-		jww.SetLogOutput(logOutput)
-	}
-
-	if threshold > 1 {
-		jww.INFO.Printf("log level set to: %s", jww.LevelTrace)
-		jww.SetStdoutThreshold(jww.LevelTrace)
-		jww.SetLogThreshold(jww.LevelTrace)
-		jww.SetFlags(log.LstdFlags | log.Lmicroseconds)
-	} else if threshold == 1 {
-		jww.INFO.Printf("log level set to: %s", jww.LevelDebug)
-		jww.SetStdoutThreshold(jww.LevelDebug)
-		jww.SetLogThreshold(jww.LevelDebug)
-		jww.SetFlags(log.LstdFlags | log.Lmicroseconds)
-	} else {
-		jww.INFO.Printf("log level set to: %s", jww.LevelInfo)
-		jww.SetStdoutThreshold(jww.LevelInfo)
-		jww.SetLogThreshold(jww.LevelInfo)
-	}
-}