From 5f80f06e7c189b9b514a24bc777acfb45b450fdf Mon Sep 17 00:00:00 2001
From: Jono Wenger <jono@elixxir.io>
Date: Fri, 20 Jan 2023 10:31:21 -0800
Subject: [PATCH] Fix flags and add more logging
---
.gitlab-ci.yml | 2 +-
emoji/emojiSet.go | 21 ++++++++-
emoji/main.go | 115 +++++++++++++++++++++++++---------------------
emoji/utils.go | 46 -------------------
4 files changed, 83 insertions(+), 101 deletions(-)
delete mode 100644 emoji/utils.go
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c2ae9a4e..4ad2d894 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -51,7 +51,7 @@ emoji-update:
script:
- go mod vendor -v
- mkdir -p release
- - go run -ldflags '-w -s' ./emoji/... -o emojiSet.json -l - -v 2
+ - go run -ldflags '-w -s' ./emoji/... -o emojiSet.json -v 2
- cp emojiSet.json release/
artifacts:
paths:
diff --git a/emoji/emojiSet.go b/emoji/emojiSet.go
index 39ab468f..2a903774 100644
--- a/emoji/emojiSet.go
+++ b/emoji/emojiSet.go
@@ -55,16 +55,19 @@ func (s *Set) SanitizeEmojiMartSet(frontendEmojiSetJson []byte) ([]byte, error)
"failed to unmarshal emoji-mart set JSON: %+v", err)
}
- jww.INFO.Printf("[SanitizeEmojiMartSet] Finding incompatible emojis...")
+ jww.INFO.Printf(
+ "Finding incompatible emojis and replacing mismatched codepoints.")
// Find all incompatible emojis in the front end set
emojisToRemove := s.findIncompatibleEmojis(&frontEndEmojiSet)
- jww.INFO.Printf("[SanitizeEmojiMartSet] Removing incompatible emojis...")
+ jww.INFO.Printf("Removing incompatible emojis.")
// Remove all incompatible emojis from the set
removeIncompatibleEmojis(&frontEndEmojiSet, emojisToRemove)
+ jww.INFO.Printf("Removed %d incompatible codepoints.", len(emojisToRemove))
+
return json.Marshal(frontEndEmojiSet)
}
@@ -79,9 +82,14 @@ func (s *Set) findIncompatibleEmojis(set *emojiMartSet) (emojisToRemove []emojiI
// Determine if the emoji's codepoint should be replaced or removed
replacement, replace := s.replace(Skin.Unified)
if replace {
+ jww.TRACE.Printf("Replaced codepoint %q with %q for emoji %q",
+ Skin.Unified, replacement, id)
newSkins = append(newSkins, replacement)
} else if !s.remove(Skin.Unified) {
newSkins = append(newSkins, Skin)
+ } else {
+ jww.TRACE.Printf("Removed codepoint %q from emoji %q",
+ Skin.Unified, id)
}
}
@@ -93,6 +101,7 @@ func (s *Set) findIncompatibleEmojis(set *emojiMartSet) (emojisToRemove []emojiI
} else {
// If all skins have been removed, then mark the emoji for removal
emojisToRemove = append(emojisToRemove, id)
+ jww.DEBUG.Printf("All skins removed for emoji %q", id)
}
}
@@ -102,8 +111,12 @@ func (s *Set) findIncompatibleEmojis(set *emojiMartSet) (emojisToRemove []emojiI
// removeIncompatibleEmojis removes all the emojis in emojisToRemove from the
// emojiMartSet set.
func removeIncompatibleEmojis(set *emojiMartSet, emojisToRemove []emojiID) {
+ jww.DEBUG.Printf(
+ "Removing %d emojis: %s", len(emojisToRemove), emojisToRemove)
+
// Remove all incompatible emojis from the emojiMartSet.Emojis list
for _, char := range emojisToRemove {
+ jww.TRACE.Printf("Removing %q from emojiMartSet.Emojis", char)
delete(set.Emojis, char)
}
@@ -115,6 +128,8 @@ func removeIncompatibleEmojis(set *emojiMartSet, emojisToRemove []emojiID) {
for _, char := range emojisToRemove {
if cat.Emojis[i] == char {
cat.Emojis = append(cat.Emojis[:i], cat.Emojis[i+1:]...)
+ jww.TRACE.Printf(
+ "Removing %q from emojiMartSet.Categories", char)
}
}
}
@@ -125,6 +140,8 @@ func removeIncompatibleEmojis(set *emojiMartSet, emojisToRemove []emojiID) {
for _, removedId := range emojisToRemove {
if id == removedId {
delete(set.Aliases, alias)
+ jww.TRACE.Printf(
+ "Removing %q from emojiMartSet.Aliases", alias)
}
}
}
diff --git a/emoji/main.go b/emoji/main.go
index faa3ee3e..135d7cab 100644
--- a/emoji/main.go
+++ b/emoji/main.go
@@ -12,8 +12,9 @@ import (
"fmt"
"github.com/spf13/cobra"
jww "github.com/spf13/jwalterweatherman"
- "github.com/spf13/viper"
"gitlab.com/xx_network/primitives/utils"
+ "io"
+ "log"
"net/http"
"os"
)
@@ -25,25 +26,24 @@ import (
// URL should be updated if new sets become available.
const emojiMartUrl = "https://raw.githubusercontent.com/missive/emoji-mart/main/packages/emoji-mart-data/sets/14/native.json"
-// Flag constants.
-const (
- sanitizedOutputFlag = "output"
- logLevelFlag = "logLevel"
- logFileFlag = "logFile"
+// Flag variables.
+var (
+ requestURL, outputPath, logFile string
+ logLevel int
)
func main() {
- if err := sanitizeEmojis.Execute(); err != nil {
+ if err := cmd.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
-// sanitizeEmojis Downloads the emoji file (from emoji-mart) and sanitizes that
+// Downloads the emoji file (from emoji-mart) and sanitizes that
// list. Sanitization removes all emojis not supported by the backend. The
-// sanitized JSON is returned via a file specified by the user. Refer to the flags
-// for details.
-var sanitizeEmojis = &cobra.Command{
+// sanitized JSON is returned via a file specified by the user. Refer to the
+// flags for details.
+var cmd = &cobra.Command{
Use: "sanitizeEmojis",
Short: "Downloads the emoji file (from emoji-mart) and sanitizes that " +
"list. Sanitization removes all emojis not supported by the backend. " +
@@ -53,20 +53,22 @@ var sanitizeEmojis = &cobra.Command{
Run: func(cmd *cobra.Command, args []string) {
// Initialize the logging if set
- if logFile := viper.GetString(logFileFlag); logFile != "" {
- initLog(viper.GetInt(logFileFlag), logFile)
+ if logFile != "" {
+ initLog(logLevel, logFile)
}
- jww.INFO.Printf("Retrieving emoji-mart JSON file...")
-
// Retrieve emoji-mart file from URL
- resp, err := http.Get(emojiMartUrl)
+ jww.INFO.Printf("Requesting file %s", requestURL)
+ resp, err := http.Get(requestURL)
if err != nil {
jww.FATAL.Panicf(
"Failed to retrieve emoji-mart JSON from URL: %+v", err)
+ } else if resp.StatusCode != http.StatusOK {
+ jww.FATAL.Panicf("Bad status: %s", resp.Status)
}
- jww.INFO.Printf("Reading emoji-mart JSON file into bytes...")
+ jww.INFO.Printf("Received HTTP response: %s", resp.Status)
+ jww.DEBUG.Printf("Response: %+v", resp)
// Read HTTP response into byte slice
var buf bytes.Buffer
@@ -79,7 +81,7 @@ var sanitizeEmojis = &cobra.Command{
}
emojiMartJson := buf.Bytes()
- jww.INFO.Printf("Sanitizing emoji-mart JSON...")
+ jww.INFO.Printf("Read %d bytes of JSON file", len(emojiMartJson))
// Sanitize the JSON file
backendSet := NewSet()
@@ -88,52 +90,61 @@ var sanitizeEmojis = &cobra.Command{
jww.FATAL.Panicf("Failed to sanitize emoji-mart list: %+v", err)
}
- jww.INFO.Printf("Outputting sanitized emoji JSON to file...")
+ jww.INFO.Printf("Sanitised JSON file.")
// Write sanitized JSON to file
- sanitizedOutputFilePath := viper.GetString(sanitizedOutputFlag)
- err = utils.WriteFileDef(sanitizedOutputFilePath, sanitizedJSON)
+ err = utils.WriteFileDef(outputPath, sanitizedJSON)
if err != nil {
jww.FATAL.Panicf(
"Failed to write sanitized emojis to filepath %s: %+v",
- sanitizedOutputFilePath, err)
+ outputPath, err)
}
+
+ jww.INFO.Printf("Wrote sanitised JSON file to %s", outputPath)
},
}
-// init is the initialization function for Cobra which defines commands
-// and flags.
+// init is the initialization function for Cobra which defines flags.
func init() {
- // Here you will define your flags and configuration settings.
- // Cobra supports persistent flags, which, if defined here,
- // will be global for your application.
- sanitizeEmojis.PersistentFlags().StringP(sanitizedOutputFlag, "o",
- "output.json",
- "File path that the sanitized JSON file will be outputted to.")
- err := viper.BindPFlag(sanitizedOutputFlag, sanitizeEmojis.PersistentFlags().
- Lookup(sanitizedOutputFlag))
- if err != nil {
- jww.FATAL.Panicf(
- "Failed to bind pf flag to %q: %+v", sanitizedOutputFlag, err)
- }
+ cmd.Flags().StringVarP(&requestURL, "url", "u", emojiMartUrl,
+ "URL to download emoji-mart JSON file.")
+ cmd.Flags().StringVarP(&outputPath, "output", "o", "output.json",
+ "Output JSON file path.")
+ cmd.Flags().StringVarP(&logFile, "log", "l", "-",
+ "Log output path. By default, logs are printed to stdout. "+
+ "To disable logging, set this to empty.")
+ cmd.Flags().IntVarP(&logLevel, "logLevel", "v", 0,
+ "Verbosity level of logging. 0 = INFO, 1 = DEBUG, 2 = TRACE")
+}
- sanitizeEmojis.PersistentFlags().StringP(logFileFlag, "l", "",
- "Path to the log output path. By default, this flag is not set "+
- "so a log will not be created unless specified.")
- err = viper.BindPFlag(logFileFlag, sanitizeEmojis.PersistentFlags().
- Lookup(logFileFlag))
- if err != nil {
- jww.FATAL.Panicf(
- "Failed to bind pf flag to %q: %+v", logFileFlag, err)
- }
+// initLog will enable JWW logging.
+func initLog(threshold int, logPath string) {
+ if logPath != "-" && logPath != "" {
+ // Disable stdout output
+ jww.SetStdoutOutput(io.Discard)
- sanitizeEmojis.PersistentFlags().IntP(logLevelFlag, "v", 0,
- "Verbosity level of logging. This defaults to 0. ")
- err = viper.BindPFlag(logLevelFlag, sanitizeEmojis.PersistentFlags().
- Lookup(logLevelFlag))
- if err != nil {
- jww.FATAL.Panicf(
- "Failed to bind pf flag to %q: %+v", logLevelFlag, err)
+ // Use log file
+ logOutput, err :=
+ os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+ if err != nil {
+ panic(err)
+ }
+ jww.SetLogOutput(logOutput)
}
+ if threshold > 1 {
+ jww.SetStdoutThreshold(jww.LevelTrace)
+ jww.SetLogThreshold(jww.LevelTrace)
+ jww.SetFlags(log.LstdFlags | log.Lmicroseconds)
+ jww.INFO.Printf("log level set to: %s", jww.LevelTrace)
+ } else if threshold == 1 {
+ jww.SetStdoutThreshold(jww.LevelDebug)
+ jww.SetLogThreshold(jww.LevelDebug)
+ jww.SetFlags(log.LstdFlags | log.Lmicroseconds)
+ jww.INFO.Printf("log level set to: %s", jww.LevelDebug)
+ } else {
+ jww.SetStdoutThreshold(jww.LevelInfo)
+ jww.SetLogThreshold(jww.LevelInfo)
+ jww.INFO.Printf("log level set to: %s", jww.LevelInfo)
+ }
}
diff --git a/emoji/utils.go b/emoji/utils.go
deleted file mode 100644
index b50d0229..00000000
--- a/emoji/utils.go
+++ /dev/null
@@ -1,46 +0,0 @@
-////////////////////////////////////////////////////////////////////////////////
-// Copyright © 2022 xx foundation //
-// //
-// Use of this source code is governed by a license that can be found in the //
-// LICENSE file. //
-////////////////////////////////////////////////////////////////////////////////
-
-package main
-
-import (
- jww "github.com/spf13/jwalterweatherman"
- "io"
- "log"
- "os"
-)
-
-// initLog will enable JWW logging.
-func initLog(threshold int, logPath string) {
- if logPath != "-" && logPath != "" {
- // Disable stdout output
- jww.SetStdoutOutput(io.Discard)
- // Use log file
- logOutput, err :=
- os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
- if err != nil {
- panic(err)
- }
- jww.SetLogOutput(logOutput)
- }
-
- if threshold > 1 {
- jww.INFO.Printf("log level set to: %s", jww.LevelTrace)
- jww.SetStdoutThreshold(jww.LevelTrace)
- jww.SetLogThreshold(jww.LevelTrace)
- jww.SetFlags(log.LstdFlags | log.Lmicroseconds)
- } else if threshold == 1 {
- jww.INFO.Printf("log level set to: %s", jww.LevelDebug)
- jww.SetStdoutThreshold(jww.LevelDebug)
- jww.SetLogThreshold(jww.LevelDebug)
- jww.SetFlags(log.LstdFlags | log.Lmicroseconds)
- } else {
- jww.INFO.Printf("log level set to: %s", jww.LevelInfo)
- jww.SetStdoutThreshold(jww.LevelInfo)
- jww.SetLogThreshold(jww.LevelInfo)
- }
-}
--
GitLab