Skip to content
Snippets Groups Projects
Commit 5f80f06e authored by Jono Wenger's avatar Jono Wenger
Browse files

Fix flags and add more logging

parent 3f29b4bd
No related branches found
No related tags found
2 merge requests!67fix for latest client release,!61Add emoji sanitizater
......@@ -51,7 +51,7 @@ emoji-update:
script:
- go mod vendor -v
- mkdir -p release
- go run -ldflags '-w -s' ./emoji/... -o emojiSet.json -l - -v 2
- go run -ldflags '-w -s' ./emoji/... -o emojiSet.json -v 2
- cp emojiSet.json release/
artifacts:
paths:
......
......@@ -55,16 +55,19 @@ func (s *Set) SanitizeEmojiMartSet(frontendEmojiSetJson []byte) ([]byte, error)
"failed to unmarshal emoji-mart set JSON: %+v", err)
}
jww.INFO.Printf("[SanitizeEmojiMartSet] Finding incompatible emojis...")
jww.INFO.Printf(
"Finding incompatible emojis and replacing mismatched codepoints.")
// Find all incompatible emojis in the front end set
emojisToRemove := s.findIncompatibleEmojis(&frontEndEmojiSet)
jww.INFO.Printf("[SanitizeEmojiMartSet] Removing incompatible emojis...")
jww.INFO.Printf("Removing incompatible emojis.")
// Remove all incompatible emojis from the set
removeIncompatibleEmojis(&frontEndEmojiSet, emojisToRemove)
jww.INFO.Printf("Removed %d incompatible codepoints.", len(emojisToRemove))
return json.Marshal(frontEndEmojiSet)
}
......@@ -79,9 +82,14 @@ func (s *Set) findIncompatibleEmojis(set *emojiMartSet) (emojisToRemove []emojiI
// Determine if the emoji's codepoint should be replaced or removed
replacement, replace := s.replace(Skin.Unified)
if replace {
jww.TRACE.Printf("Replaced codepoint %q with %q for emoji %q",
Skin.Unified, replacement, id)
newSkins = append(newSkins, replacement)
} else if !s.remove(Skin.Unified) {
newSkins = append(newSkins, Skin)
} else {
jww.TRACE.Printf("Removed codepoint %q from emoji %q",
Skin.Unified, id)
}
}
......@@ -93,6 +101,7 @@ func (s *Set) findIncompatibleEmojis(set *emojiMartSet) (emojisToRemove []emojiI
} else {
// If all skins have been removed, then mark the emoji for removal
emojisToRemove = append(emojisToRemove, id)
jww.DEBUG.Printf("All skins removed for emoji %q", id)
}
}
......@@ -102,8 +111,12 @@ func (s *Set) findIncompatibleEmojis(set *emojiMartSet) (emojisToRemove []emojiI
// removeIncompatibleEmojis removes all the emojis in emojisToRemove from the
// emojiMartSet set.
func removeIncompatibleEmojis(set *emojiMartSet, emojisToRemove []emojiID) {
jww.DEBUG.Printf(
"Removing %d emojis: %s", len(emojisToRemove), emojisToRemove)
// Remove all incompatible emojis from the emojiMartSet.Emojis list
for _, char := range emojisToRemove {
jww.TRACE.Printf("Removing %q from emojiMartSet.Emojis", char)
delete(set.Emojis, char)
}
......@@ -115,6 +128,8 @@ func removeIncompatibleEmojis(set *emojiMartSet, emojisToRemove []emojiID) {
for _, char := range emojisToRemove {
if cat.Emojis[i] == char {
cat.Emojis = append(cat.Emojis[:i], cat.Emojis[i+1:]...)
jww.TRACE.Printf(
"Removing %q from emojiMartSet.Categories", char)
}
}
}
......@@ -125,6 +140,8 @@ func removeIncompatibleEmojis(set *emojiMartSet, emojisToRemove []emojiID) {
for _, removedId := range emojisToRemove {
if id == removedId {
delete(set.Aliases, alias)
jww.TRACE.Printf(
"Removing %q from emojiMartSet.Aliases", alias)
}
}
}
......
......@@ -12,8 +12,9 @@ import (
"fmt"
"github.com/spf13/cobra"
jww "github.com/spf13/jwalterweatherman"
"github.com/spf13/viper"
"gitlab.com/xx_network/primitives/utils"
"io"
"log"
"net/http"
"os"
)
......@@ -25,25 +26,24 @@ import (
// URL should be updated if new sets become available.
const emojiMartUrl = "https://raw.githubusercontent.com/missive/emoji-mart/main/packages/emoji-mart-data/sets/14/native.json"
// Flag constants.
const (
sanitizedOutputFlag = "output"
logLevelFlag = "logLevel"
logFileFlag = "logFile"
// Flag variables.
var (
requestURL, outputPath, logFile string
logLevel int
)
func main() {
if err := sanitizeEmojis.Execute(); err != nil {
if err := cmd.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
// sanitizeEmojis Downloads the emoji file (from emoji-mart) and sanitizes that
// Downloads the emoji file (from emoji-mart) and sanitizes that
// list. Sanitization removes all emojis not supported by the backend. The
// sanitized JSON is returned via a file specified by the user. Refer to the flags
// for details.
var sanitizeEmojis = &cobra.Command{
// sanitized JSON is returned via a file specified by the user. Refer to the
// flags for details.
var cmd = &cobra.Command{
Use: "sanitizeEmojis",
Short: "Downloads the emoji file (from emoji-mart) and sanitizes that " +
"list. Sanitization removes all emojis not supported by the backend. " +
......@@ -53,20 +53,22 @@ var sanitizeEmojis = &cobra.Command{
Run: func(cmd *cobra.Command, args []string) {
// Initialize the logging if set
if logFile := viper.GetString(logFileFlag); logFile != "" {
initLog(viper.GetInt(logFileFlag), logFile)
if logFile != "" {
initLog(logLevel, logFile)
}
jww.INFO.Printf("Retrieving emoji-mart JSON file...")
// Retrieve emoji-mart file from URL
resp, err := http.Get(emojiMartUrl)
jww.INFO.Printf("Requesting file %s", requestURL)
resp, err := http.Get(requestURL)
if err != nil {
jww.FATAL.Panicf(
"Failed to retrieve emoji-mart JSON from URL: %+v", err)
} else if resp.StatusCode != http.StatusOK {
jww.FATAL.Panicf("Bad status: %s", resp.Status)
}
jww.INFO.Printf("Reading emoji-mart JSON file into bytes...")
jww.INFO.Printf("Received HTTP response: %s", resp.Status)
jww.DEBUG.Printf("Response: %+v", resp)
// Read HTTP response into byte slice
var buf bytes.Buffer
......@@ -79,7 +81,7 @@ var sanitizeEmojis = &cobra.Command{
}
emojiMartJson := buf.Bytes()
jww.INFO.Printf("Sanitizing emoji-mart JSON...")
jww.INFO.Printf("Read %d bytes of JSON file", len(emojiMartJson))
// Sanitize the JSON file
backendSet := NewSet()
......@@ -88,52 +90,61 @@ var sanitizeEmojis = &cobra.Command{
jww.FATAL.Panicf("Failed to sanitize emoji-mart list: %+v", err)
}
jww.INFO.Printf("Outputting sanitized emoji JSON to file...")
jww.INFO.Printf("Sanitised JSON file.")
// Write sanitized JSON to file
sanitizedOutputFilePath := viper.GetString(sanitizedOutputFlag)
err = utils.WriteFileDef(sanitizedOutputFilePath, sanitizedJSON)
err = utils.WriteFileDef(outputPath, sanitizedJSON)
if err != nil {
jww.FATAL.Panicf(
"Failed to write sanitized emojis to filepath %s: %+v",
sanitizedOutputFilePath, err)
outputPath, err)
}
jww.INFO.Printf("Wrote sanitised JSON file to %s", outputPath)
},
}
// init is the initialization function for Cobra which defines commands
// and flags.
// init is the initialization function for Cobra which defines flags.
func init() {
// Here you will define your flags and configuration settings.
// Cobra supports persistent flags, which, if defined here,
// will be global for your application.
sanitizeEmojis.PersistentFlags().StringP(sanitizedOutputFlag, "o",
"output.json",
"File path that the sanitized JSON file will be outputted to.")
err := viper.BindPFlag(sanitizedOutputFlag, sanitizeEmojis.PersistentFlags().
Lookup(sanitizedOutputFlag))
if err != nil {
jww.FATAL.Panicf(
"Failed to bind pf flag to %q: %+v", sanitizedOutputFlag, err)
cmd.Flags().StringVarP(&requestURL, "url", "u", emojiMartUrl,
"URL to download emoji-mart JSON file.")
cmd.Flags().StringVarP(&outputPath, "output", "o", "output.json",
"Output JSON file path.")
cmd.Flags().StringVarP(&logFile, "log", "l", "-",
"Log output path. By default, logs are printed to stdout. "+
"To disable logging, set this to empty.")
cmd.Flags().IntVarP(&logLevel, "logLevel", "v", 0,
"Verbosity level of logging. 0 = INFO, 1 = DEBUG, 2 = TRACE")
}
sanitizeEmojis.PersistentFlags().StringP(logFileFlag, "l", "",
"Path to the log output path. By default, this flag is not set "+
"so a log will not be created unless specified.")
err = viper.BindPFlag(logFileFlag, sanitizeEmojis.PersistentFlags().
Lookup(logFileFlag))
if err != nil {
jww.FATAL.Panicf(
"Failed to bind pf flag to %q: %+v", logFileFlag, err)
}
// initLog will enable JWW logging.
func initLog(threshold int, logPath string) {
if logPath != "-" && logPath != "" {
// Disable stdout output
jww.SetStdoutOutput(io.Discard)
sanitizeEmojis.PersistentFlags().IntP(logLevelFlag, "v", 0,
"Verbosity level of logging. This defaults to 0. ")
err = viper.BindPFlag(logLevelFlag, sanitizeEmojis.PersistentFlags().
Lookup(logLevelFlag))
// Use log file
logOutput, err :=
os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
jww.FATAL.Panicf(
"Failed to bind pf flag to %q: %+v", logLevelFlag, err)
panic(err)
}
jww.SetLogOutput(logOutput)
}
if threshold > 1 {
jww.SetStdoutThreshold(jww.LevelTrace)
jww.SetLogThreshold(jww.LevelTrace)
jww.SetFlags(log.LstdFlags | log.Lmicroseconds)
jww.INFO.Printf("log level set to: %s", jww.LevelTrace)
} else if threshold == 1 {
jww.SetStdoutThreshold(jww.LevelDebug)
jww.SetLogThreshold(jww.LevelDebug)
jww.SetFlags(log.LstdFlags | log.Lmicroseconds)
jww.INFO.Printf("log level set to: %s", jww.LevelDebug)
} else {
jww.SetStdoutThreshold(jww.LevelInfo)
jww.SetLogThreshold(jww.LevelInfo)
jww.INFO.Printf("log level set to: %s", jww.LevelInfo)
}
}
////////////////////////////////////////////////////////////////////////////////
// Copyright © 2022 xx foundation //
// //
// Use of this source code is governed by a license that can be found in the //
// LICENSE file. //
////////////////////////////////////////////////////////////////////////////////
package main
import (
jww "github.com/spf13/jwalterweatherman"
"io"
"log"
"os"
)
// initLog will enable JWW logging.
func initLog(threshold int, logPath string) {
if logPath != "-" && logPath != "" {
// Disable stdout output
jww.SetStdoutOutput(io.Discard)
// Use log file
logOutput, err :=
os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
panic(err)
}
jww.SetLogOutput(logOutput)
}
if threshold > 1 {
jww.INFO.Printf("log level set to: %s", jww.LevelTrace)
jww.SetStdoutThreshold(jww.LevelTrace)
jww.SetLogThreshold(jww.LevelTrace)
jww.SetFlags(log.LstdFlags | log.Lmicroseconds)
} else if threshold == 1 {
jww.INFO.Printf("log level set to: %s", jww.LevelDebug)
jww.SetStdoutThreshold(jww.LevelDebug)
jww.SetLogThreshold(jww.LevelDebug)
jww.SetFlags(log.LstdFlags | log.Lmicroseconds)
} else {
jww.INFO.Printf("log level set to: %s", jww.LevelInfo)
jww.SetStdoutThreshold(jww.LevelInfo)
jww.SetLogThreshold(jww.LevelInfo)
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment