Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
X
xxdk-wasm
Manage
Activity
Members
Labels
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Terraform modules
Analyze
Contributor analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
elixxir
xxdk-wasm
Commits
5f80f06e
Commit
5f80f06e
authored
Jan 20, 2023
by
Jono Wenger
Browse files
Options
Downloads
Patches
Plain Diff
Fix flags and add more logging
parent
3f29b4bd
No related branches found
No related tags found
2 merge requests
!67
fix for latest client release
,
!61
Add emoji sanitizater
Changes
4
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
.gitlab-ci.yml
+1
-1
1 addition, 1 deletion
.gitlab-ci.yml
emoji/emojiSet.go
+19
-2
19 additions, 2 deletions
emoji/emojiSet.go
emoji/main.go
+63
-52
63 additions, 52 deletions
emoji/main.go
emoji/utils.go
+0
-46
0 additions, 46 deletions
emoji/utils.go
with
83 additions
and
101 deletions
.gitlab-ci.yml
+
1
−
1
View file @
5f80f06e
...
...
@@ -51,7 +51,7 @@ emoji-update:
script
:
-
go mod vendor -v
-
mkdir -p release
-
go run -ldflags '-w -s' ./emoji/... -o emojiSet.json
-l -
-v
2
-
go run -ldflags '-w -s' ./emoji/... -o emojiSet.json -v
2
-
cp emojiSet.json release/
artifacts
:
paths
:
...
...
This diff is collapsed.
Click to expand it.
emoji/emojiSet.go
+
19
−
2
View file @
5f80f06e
...
...
@@ -55,16 +55,19 @@ func (s *Set) SanitizeEmojiMartSet(frontendEmojiSetJson []byte) ([]byte, error)
"failed to unmarshal emoji-mart set JSON: %+v"
,
err
)
}
jww
.
INFO
.
Printf
(
"[SanitizeEmojiMartSet] Finding incompatible emojis..."
)
jww
.
INFO
.
Printf
(
"Finding incompatible emojis and replacing mismatched codepoints."
)
// Find all incompatible emojis in the front end set
emojisToRemove
:=
s
.
findIncompatibleEmojis
(
&
frontEndEmojiSet
)
jww
.
INFO
.
Printf
(
"
[SanitizeEmojiMartSet]
Removing incompatible emojis.
..
"
)
jww
.
INFO
.
Printf
(
"Removing incompatible emojis."
)
// Remove all incompatible emojis from the set
removeIncompatibleEmojis
(
&
frontEndEmojiSet
,
emojisToRemove
)
jww
.
INFO
.
Printf
(
"Removed %d incompatible codepoints."
,
len
(
emojisToRemove
))
return
json
.
Marshal
(
frontEndEmojiSet
)
}
...
...
@@ -79,9 +82,14 @@ func (s *Set) findIncompatibleEmojis(set *emojiMartSet) (emojisToRemove []emojiI
// Determine if the emoji's codepoint should be replaced or removed
replacement
,
replace
:=
s
.
replace
(
Skin
.
Unified
)
if
replace
{
jww
.
TRACE
.
Printf
(
"Replaced codepoint %q with %q for emoji %q"
,
Skin
.
Unified
,
replacement
,
id
)
newSkins
=
append
(
newSkins
,
replacement
)
}
else
if
!
s
.
remove
(
Skin
.
Unified
)
{
newSkins
=
append
(
newSkins
,
Skin
)
}
else
{
jww
.
TRACE
.
Printf
(
"Removed codepoint %q from emoji %q"
,
Skin
.
Unified
,
id
)
}
}
...
...
@@ -93,6 +101,7 @@ func (s *Set) findIncompatibleEmojis(set *emojiMartSet) (emojisToRemove []emojiI
}
else
{
// If all skins have been removed, then mark the emoji for removal
emojisToRemove
=
append
(
emojisToRemove
,
id
)
jww
.
DEBUG
.
Printf
(
"All skins removed for emoji %q"
,
id
)
}
}
...
...
@@ -102,8 +111,12 @@ func (s *Set) findIncompatibleEmojis(set *emojiMartSet) (emojisToRemove []emojiI
// removeIncompatibleEmojis removes all the emojis in emojisToRemove from the
// emojiMartSet set.
func
removeIncompatibleEmojis
(
set
*
emojiMartSet
,
emojisToRemove
[]
emojiID
)
{
jww
.
DEBUG
.
Printf
(
"Removing %d emojis: %s"
,
len
(
emojisToRemove
),
emojisToRemove
)
// Remove all incompatible emojis from the emojiMartSet.Emojis list
for
_
,
char
:=
range
emojisToRemove
{
jww
.
TRACE
.
Printf
(
"Removing %q from emojiMartSet.Emojis"
,
char
)
delete
(
set
.
Emojis
,
char
)
}
...
...
@@ -115,6 +128,8 @@ func removeIncompatibleEmojis(set *emojiMartSet, emojisToRemove []emojiID) {
for
_
,
char
:=
range
emojisToRemove
{
if
cat
.
Emojis
[
i
]
==
char
{
cat
.
Emojis
=
append
(
cat
.
Emojis
[
:
i
],
cat
.
Emojis
[
i
+
1
:
]
...
)
jww
.
TRACE
.
Printf
(
"Removing %q from emojiMartSet.Categories"
,
char
)
}
}
}
...
...
@@ -125,6 +140,8 @@ func removeIncompatibleEmojis(set *emojiMartSet, emojisToRemove []emojiID) {
for
_
,
removedId
:=
range
emojisToRemove
{
if
id
==
removedId
{
delete
(
set
.
Aliases
,
alias
)
jww
.
TRACE
.
Printf
(
"Removing %q from emojiMartSet.Aliases"
,
alias
)
}
}
}
...
...
This diff is collapsed.
Click to expand it.
emoji/main.go
+
63
−
52
View file @
5f80f06e
...
...
@@ -12,8 +12,9 @@ import (
"fmt"
"github.com/spf13/cobra"
jww
"github.com/spf13/jwalterweatherman"
"github.com/spf13/viper"
"gitlab.com/xx_network/primitives/utils"
"io"
"log"
"net/http"
"os"
)
...
...
@@ -25,25 +26,24 @@ import (
// URL should be updated if new sets become available.
const
emojiMartUrl
=
"https://raw.githubusercontent.com/missive/emoji-mart/main/packages/emoji-mart-data/sets/14/native.json"
// Flag constants.
const
(
sanitizedOutputFlag
=
"output"
logLevelFlag
=
"logLevel"
logFileFlag
=
"logFile"
// Flag variables.
var
(
requestURL
,
outputPath
,
logFile
string
logLevel
int
)
func
main
()
{
if
err
:=
sanitizeEmojis
.
Execute
();
err
!=
nil
{
if
err
:=
cmd
.
Execute
();
err
!=
nil
{
fmt
.
Println
(
err
)
os
.
Exit
(
1
)
}
}
//
sanitizeEmojis
Downloads the emoji file (from emoji-mart) and sanitizes that
// Downloads the emoji file (from emoji-mart) and sanitizes that
// list. Sanitization removes all emojis not supported by the backend. The
// sanitized JSON is returned via a file specified by the user. Refer to the
flags
// for details.
var
sanitizeEmojis
=
&
cobra
.
Command
{
// sanitized JSON is returned via a file specified by the user. Refer to the
//
flags
for details.
var
cmd
=
&
cobra
.
Command
{
Use
:
"sanitizeEmojis"
,
Short
:
"Downloads the emoji file (from emoji-mart) and sanitizes that "
+
"list. Sanitization removes all emojis not supported by the backend. "
+
...
...
@@ -53,20 +53,22 @@ var sanitizeEmojis = &cobra.Command{
Run
:
func
(
cmd
*
cobra
.
Command
,
args
[]
string
)
{
// Initialize the logging if set
if
logFile
:=
viper
.
GetString
(
logFileFlag
);
logFile
!=
""
{
initLog
(
viper
.
GetInt
(
logFileFlag
)
,
logFile
)
if
logFile
!=
""
{
initLog
(
logLevel
,
logFile
)
}
jww
.
INFO
.
Printf
(
"Retrieving emoji-mart JSON file..."
)
// Retrieve emoji-mart file from URL
resp
,
err
:=
http
.
Get
(
emojiMartUrl
)
jww
.
INFO
.
Printf
(
"Requesting file %s"
,
requestURL
)
resp
,
err
:=
http
.
Get
(
requestURL
)
if
err
!=
nil
{
jww
.
FATAL
.
Panicf
(
"Failed to retrieve emoji-mart JSON from URL: %+v"
,
err
)
}
else
if
resp
.
StatusCode
!=
http
.
StatusOK
{
jww
.
FATAL
.
Panicf
(
"Bad status: %s"
,
resp
.
Status
)
}
jww
.
INFO
.
Printf
(
"Reading emoji-mart JSON file into bytes..."
)
jww
.
INFO
.
Printf
(
"Received HTTP response: %s"
,
resp
.
Status
)
jww
.
DEBUG
.
Printf
(
"Response: %+v"
,
resp
)
// Read HTTP response into byte slice
var
buf
bytes
.
Buffer
...
...
@@ -79,7 +81,7 @@ var sanitizeEmojis = &cobra.Command{
}
emojiMartJson
:=
buf
.
Bytes
()
jww
.
INFO
.
Printf
(
"
Sanitizing
emoji
-m
art
JSON..."
)
jww
.
INFO
.
Printf
(
"
Read %d bytes of JSON file"
,
len
(
emoji
M
art
Json
)
)
// Sanitize the JSON file
backendSet
:=
NewSet
()
...
...
@@ -88,52 +90,61 @@ var sanitizeEmojis = &cobra.Command{
jww
.
FATAL
.
Panicf
(
"Failed to sanitize emoji-mart list: %+v"
,
err
)
}
jww
.
INFO
.
Printf
(
"
Outputting s
aniti
z
ed
emoji
JSON
to
file.
..
"
)
jww
.
INFO
.
Printf
(
"
S
aniti
s
ed JSON file."
)
// Write sanitized JSON to file
sanitizedOutputFilePath
:=
viper
.
GetString
(
sanitizedOutputFlag
)
err
=
utils
.
WriteFileDef
(
sanitizedOutputFilePath
,
sanitizedJSON
)
err
=
utils
.
WriteFileDef
(
outputPath
,
sanitizedJSON
)
if
err
!=
nil
{
jww
.
FATAL
.
Panicf
(
"Failed to write sanitized emojis to filepath %s: %+v"
,
sanitizedO
utput
File
Path
,
err
)
o
utputPath
,
err
)
}
jww
.
INFO
.
Printf
(
"Wrote sanitised JSON file to %s"
,
outputPath
)
},
}
// init is the initialization function for Cobra which defines commands
// and flags.
// init is the initialization function for Cobra which defines flags.
func
init
()
{
// Here you will define your flags and configuration settings.
// Cobra supports persistent flags, which, if defined here,
// will be global for your application.
sanitizeEmojis
.
PersistentFlags
()
.
StringP
(
sanitizedOutputFlag
,
"o"
,
"output.json"
,
"File path that the sanitized JSON file will be outputted to."
)
err
:=
viper
.
BindPFlag
(
sanitizedOutputFlag
,
sanitizeEmojis
.
PersistentFlags
()
.
Lookup
(
sanitizedOutputFlag
))
if
err
!=
nil
{
jww
.
FATAL
.
Panicf
(
"Failed to bind pf flag to %q: %+v"
,
sanitizedOutputFlag
,
err
)
cmd
.
Flags
()
.
StringVarP
(
&
requestURL
,
"url"
,
"u"
,
emojiMartUrl
,
"URL to download emoji-mart JSON file."
)
cmd
.
Flags
()
.
StringVarP
(
&
outputPath
,
"output"
,
"o"
,
"output.json"
,
"Output JSON file path."
)
cmd
.
Flags
()
.
StringVarP
(
&
logFile
,
"log"
,
"l"
,
"-"
,
"Log output path. By default, logs are printed to stdout. "
+
"To disable logging, set this to empty."
)
cmd
.
Flags
()
.
IntVarP
(
&
logLevel
,
"logLevel"
,
"v"
,
0
,
"Verbosity level of logging. 0 = INFO, 1 = DEBUG, 2 = TRACE"
)
}
sanitizeEmojis
.
PersistentFlags
()
.
StringP
(
logFileFlag
,
"l"
,
""
,
"Path to the log output path. By default, this flag is not set "
+
"so a log will not be created unless specified."
)
err
=
viper
.
BindPFlag
(
logFileFlag
,
sanitizeEmojis
.
PersistentFlags
()
.
Lookup
(
logFileFlag
))
if
err
!=
nil
{
jww
.
FATAL
.
Panicf
(
"Failed to bind pf flag to %q: %+v"
,
logFileFlag
,
err
)
}
// initLog will enable JWW logging.
func
initLog
(
threshold
int
,
logPath
string
)
{
if
logPath
!=
"-"
&&
logPath
!=
""
{
// Disable stdout output
jww
.
SetStdoutOutput
(
io
.
Discard
)
sanitizeEmojis
.
PersistentFlags
()
.
IntP
(
logLevelFlag
,
"v"
,
0
,
"Verbosity level of logging. This defaults to 0. "
)
err
=
viper
.
BindPFlag
(
logLevelFlag
,
sanitizeEmojis
.
PersistentFlags
()
.
Lookup
(
logLevelFlag
))
// Use log file
logOutput
,
err
:=
os
.
OpenFile
(
logPath
,
os
.
O_APPEND
|
os
.
O_CREATE
|
os
.
O_WRONLY
,
0644
)
if
err
!=
nil
{
jww
.
FATAL
.
Panicf
(
"Failed to bind pf flag to %q: %+v"
,
logLevelFlag
,
err
)
panic
(
err
)
}
jww
.
SetLogOutput
(
logOutput
)
}
if
threshold
>
1
{
jww
.
SetStdoutThreshold
(
jww
.
LevelTrace
)
jww
.
SetLogThreshold
(
jww
.
LevelTrace
)
jww
.
SetFlags
(
log
.
LstdFlags
|
log
.
Lmicroseconds
)
jww
.
INFO
.
Printf
(
"log level set to: %s"
,
jww
.
LevelTrace
)
}
else
if
threshold
==
1
{
jww
.
SetStdoutThreshold
(
jww
.
LevelDebug
)
jww
.
SetLogThreshold
(
jww
.
LevelDebug
)
jww
.
SetFlags
(
log
.
LstdFlags
|
log
.
Lmicroseconds
)
jww
.
INFO
.
Printf
(
"log level set to: %s"
,
jww
.
LevelDebug
)
}
else
{
jww
.
SetStdoutThreshold
(
jww
.
LevelInfo
)
jww
.
SetLogThreshold
(
jww
.
LevelInfo
)
jww
.
INFO
.
Printf
(
"log level set to: %s"
,
jww
.
LevelInfo
)
}
}
This diff is collapsed.
Click to expand it.
emoji/utils.go
deleted
100644 → 0
+
0
−
46
View file @
3f29b4bd
////////////////////////////////////////////////////////////////////////////////
// Copyright © 2022 xx foundation //
// //
// Use of this source code is governed by a license that can be found in the //
// LICENSE file. //
////////////////////////////////////////////////////////////////////////////////
package
main
import
(
jww
"github.com/spf13/jwalterweatherman"
"io"
"log"
"os"
)
// initLog will enable JWW logging.
func
initLog
(
threshold
int
,
logPath
string
)
{
if
logPath
!=
"-"
&&
logPath
!=
""
{
// Disable stdout output
jww
.
SetStdoutOutput
(
io
.
Discard
)
// Use log file
logOutput
,
err
:=
os
.
OpenFile
(
logPath
,
os
.
O_APPEND
|
os
.
O_CREATE
|
os
.
O_WRONLY
,
0644
)
if
err
!=
nil
{
panic
(
err
)
}
jww
.
SetLogOutput
(
logOutput
)
}
if
threshold
>
1
{
jww
.
INFO
.
Printf
(
"log level set to: %s"
,
jww
.
LevelTrace
)
jww
.
SetStdoutThreshold
(
jww
.
LevelTrace
)
jww
.
SetLogThreshold
(
jww
.
LevelTrace
)
jww
.
SetFlags
(
log
.
LstdFlags
|
log
.
Lmicroseconds
)
}
else
if
threshold
==
1
{
jww
.
INFO
.
Printf
(
"log level set to: %s"
,
jww
.
LevelDebug
)
jww
.
SetStdoutThreshold
(
jww
.
LevelDebug
)
jww
.
SetLogThreshold
(
jww
.
LevelDebug
)
jww
.
SetFlags
(
log
.
LstdFlags
|
log
.
Lmicroseconds
)
}
else
{
jww
.
INFO
.
Printf
(
"log level set to: %s"
,
jww
.
LevelInfo
)
jww
.
SetStdoutThreshold
(
jww
.
LevelInfo
)
jww
.
SetLogThreshold
(
jww
.
LevelInfo
)
}
}
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment