From 8c6d01e497b1a957876484f8c453b6de6ee17948 Mon Sep 17 00:00:00 2001 From: thisisommore <ommore501@gmail.com> Date: Wed, 5 Mar 2025 13:49:07 +0000 Subject: [PATCH 1/6] Add ExternalStorage implementation --- storage/externalStorage.go | 333 +++++++++++++++++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 storage/externalStorage.go diff --git a/storage/externalStorage.go b/storage/externalStorage.go new file mode 100644 index 0000000..1903616 --- /dev/null +++ b/storage/externalStorage.go @@ -0,0 +1,333 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright © 2022 xx foundation // +// // +// Use of this source code is governed by a license that can be found in the // +// LICENSE file. // +//////////////////////////////////////////////////////////////////////////////// + +//go:build js && wasm + +package storage + +import ( + "os" + "strings" + "syscall/js" + + "github.com/Max-Sum/base32768" + + "gitlab.com/elixxir/wasm-utils/exception" + "gitlab.com/elixxir/wasm-utils/utils" +) + +// externalStorageWasmPrefix is prefixed to every keyName saved to external storage by +// externalStorage. It allows the identification and deletion of keys only created +// by this WASM binary while ignoring keys made by other scripts on the same +// page. +// +// The chosen prefix is two characters, that when converted to UTF16, take up 4 +// bytes without any zeros to make them more unique. +const externalStorageWasmPrefix = "🞮🞮" + +// ExternalStorage defines an interface for setting persistent state in a KV format +// specifically for web-based implementations. +type ExternalStorage interface { + // Get decodes and returns the value from the external storage given its key + // name. Returns os.ErrNotExist if the key does not exist. + Get(key string) ([]byte, error) + + // Set encodes the bytes to a string and adds them to external storage at the + // given key name. Returns an error if external storage quota has been reached. + Set(key string, value []byte) error + + // RemoveItem removes a key's value from external storage given its name. If + // there is no item with the given key, this function does nothing. + RemoveItem(keyName string) error + + // Clear clears all the keys in storage. Returns the number of keys cleared and any error. + Clear() (int, error) + + // ClearPrefix clears all keys with the given prefix. Returns the number of + // keys cleared and any error. + ClearPrefix(prefix string) (int, error) + + // Key returns the name of the nth key in externalStorage. Returns + // os.ErrNotExist if the key does not exist. The order of keys is not + // defined. + Key(n int) (string, error) + + // Keys returns a list of all key names in external storage. + Keys() ([]string, error) + + // Length returns the number of keys in externalStorage. + Length() (int, error) + + // ExternalStorageUNSAFE returns the underlying external storage wrapper. This can + // be UNSAFE and should only be used if you know what you are doing. + // + // The returned wrapper wraps all the functions and fields on the Javascript + // havenStorage object to handle type conversions and errors. But it does + // not decode/sanitize the inputs/outputs or track entries using the prefix + // system. If using it, make sure all key names and values can be converted + // to valid UCS-2 strings. + ExternalStorageUNSAFE() *HavenStorageJS +} + +// externalStorage contains the js.Value representation of havenStorage. +type externalStorage struct { + // The Javascript value containing the havenStorage object + v *HavenStorageJS + + // The prefix appended to each key name. This is so that all keys created by + // this structure can be deleted without affecting other keys in external + // storage. + prefix string +} + +// jsStorage is the global that stores Javascript as window.havenStorage. +var jsExternalStorage ExternalStorage = newExternalStorage(externalStorageWasmPrefix) + +// newExternalStorage creates a new externalStorage object with the specified prefix. +func newExternalStorage(prefix string) *externalStorage { + return &externalStorage{ + v: &HavenStorageJS{js.Global().Get("havenStorage")}, + prefix: prefix, + } +} + +// GetExternalStorage returns Javascript's external storage. +func GetExternalStorage() ExternalStorage { + return jsExternalStorage +} + +// Get decodes and returns the value from the external storage given its key +// name. Returns os.ErrNotExist if the key does not exist. +func (ls *externalStorage) Get(keyName string) ([]byte, error) { + value, err := ls.v.GetItem(ls.prefix + keyName) + if err != nil { + return nil, err + } + + return base32768.SafeEncoding.DecodeString(value) +} + +// Set encodes the bytes to a string and adds them to external storage at the +// given key name. Returns an error if external storage quota has been reached. +func (ls *externalStorage) Set(keyName string, keyValue []byte) error { + encoded := base32768.SafeEncoding.EncodeToString(keyValue) + return ls.v.SetItem(ls.prefix+keyName, encoded) +} + +// RemoveItem removes a key's value from external storage given its name. If there +// is no item with the given key, this function does nothing. +func (ls *externalStorage) RemoveItem(keyName string) error { + return ls.v.RemoveItem(ls.prefix + keyName) +} + +// Clear clears all the keys in storage. Returns the number of keys cleared and any error. +func (ls *externalStorage) Clear() (int, error) { + // Get a copy of all key names at once + keys, err := ls.v.KeysPrefix(ls.prefix) + if err != nil { + return 0, err + } + + // Loop through each key + for _, keyName := range keys { + if err := ls.RemoveItem(keyName); err != nil { + return 0, err + } + } + + return len(keys), nil +} + +// ClearPrefix clears all keys with the given prefix. Returns the number of +// keys cleared and any error. +func (ls *externalStorage) ClearPrefix(prefix string) (int, error) { + // Get a copy of all key names at once + keys, err := ls.v.KeysPrefix(ls.prefix + prefix) + if err != nil { + return 0, err + } + + // Loop through each key + for _, keyName := range keys { + if err := ls.RemoveItem(prefix + keyName); err != nil { + return 0, err + } + } + + return len(keys), nil +} + +// Key returns the name of the nth key in externalStorage. Return [os.ErrNotExist] +// if the key does not exist. The order of keys is not defined. +func (ls *externalStorage) Key(n int) (string, error) { + keyName, err := ls.v.Key(n) + if err != nil { + return "", err + } + return strings.TrimPrefix(keyName, ls.prefix), nil +} + +// Keys returns a list of all key names in external storage. +func (ls *externalStorage) Keys() ([]string, error) { + keys, err := ls.v.KeysPrefix(ls.prefix) + if err != nil { + return nil, err + } + return keys, nil +} + +// Length returns the number of keys in externalStorage. +func (ls *externalStorage) Length() (int, error) { + length, err := ls.v.Length() + if err != nil { + return 0, err + } + return length, nil +} + +// ExternalStorageUNSAFE returns the underlying external storage wrapper. This can be +// UNSAFE and should only be used if you know what you are doing. +// +// The returned wrapper wraps all the functions and fields on the Javascript +// havenStorage object to handle type conversions and errors. But it does not +// decode/sanitize the inputs/outputs or track entries using the prefix system. +// If using it, make sure all key names and values can be converted to valid +// UCS-2 strings. +func (ls *externalStorage) ExternalStorageUNSAFE() *HavenStorageJS { + return ls.v +} + +//////////////////////////////////////////////////////////////////////////////// +// Javascript Wrappers // +//////////////////////////////////////////////////////////////////////////////// + +// HavenStorageJS stores the Javascript window.havenStorage object and wraps all +// of its methods and fields to handle type conversations and errors. +type HavenStorageJS struct { + js.Value +} + +// GetItem returns the value from the external storage given its key name. Returns +// [os.ErrNotExist] if the key does not exist. +// +// Doc: https://developer.mozilla.org/en-US/docs/Web/API/Storage/getItem +func (ls *HavenStorageJS) GetItem(keyName string) (keyValue string, err error) { + defer exception.Catch(&err) + promise := ls.Call("getItem", keyName) + result, jsErr := utils.Await(promise) + if jsErr != nil { + return "", js.Error{Value: jsErr[0]} + } + if result[0].IsNull() { + return "", os.ErrNotExist + } + return result[0].String(), nil +} + +// SetItem adds the value to external storage at the given key name. Returns an +// error if external storage quota has been reached. +// +// Doc: https://developer.mozilla.org/en-US/docs/Web/API/Storage/setItem +func (ls *HavenStorageJS) SetItem(keyName, keyValue string) (err error) { + defer exception.Catch(&err) + promise := ls.Call("setItem", keyName, keyValue) + _, jsErr := utils.Await(promise) + if jsErr != nil { + return js.Error{Value: jsErr[0]} + } + return nil +} + +// RemoveItem removes a key's value from external storage given its name. If there +// is no item with the given key, this function does nothing. +// +// Doc: https://developer.mozilla.org/en-US/docs/Web/API/Storage/removeItem +func (ls *HavenStorageJS) RemoveItem(keyName string) error { + promise := ls.Call("removeItem", keyName) + _, jsErr := utils.Await(promise) + if jsErr != nil { + return js.Error{Value: jsErr[0]} + } + return nil +} + +// Clear clears all the keys in storage. +// +// Doc: https://developer.mozilla.org/en-US/docs/Web/API/Storage/clear +func (ls *HavenStorageJS) Clear() error { + promise := ls.Call("clear") + _, jsErr := utils.Await(promise) + if jsErr != nil { + return js.Error{Value: jsErr[0]} + } + return nil +} + +// Key returns the name of the nth key in externalStorage. Return [os.ErrNotExist] +// if the key does not exist. The order of keys is not defined. +// +// Doc: https://developer.mozilla.org/en-US/docs/Web/API/Storage/key +func (ls *HavenStorageJS) Key(n int) (keyName string, err error) { + defer exception.Catch(&err) + promise := ls.Call("key", n) + result, jsErr := utils.Await(promise) + if jsErr != nil { + return "", js.Error{Value: jsErr[0]} + } + if result[0].IsNull() { + return "", os.ErrNotExist + } + return result[0].String(), nil +} + +// Keys returns a list of all key names in external storage. +func (ls *HavenStorageJS) Keys() ([]string, error) { + promise := ls.Call("keys") + result, jsErr := utils.Await(promise) + if jsErr != nil { + return []string{}, js.Error{Value: jsErr[0]} + } + + keysJS := result[0] + keys := make([]string, keysJS.Length()) + for i := range keys { + keys[i] = keysJS.Index(i).String() + } + return keys, nil +} + +// KeysPrefix returns a list of all key names in external storage with the given +// prefix and trims the prefix from each key name. +func (ls *HavenStorageJS) KeysPrefix(prefix string) ([]string, error) { + promise := ls.Call("keys") + result, jsErr := utils.Await(promise) + if jsErr != nil { + return []string{}, js.Error{Value: jsErr[0]} + } + + keysJS := result[0] + keys := make([]string, 0, keysJS.Length()) + for i := 0; i < keysJS.Length(); i++ { + keyName := keysJS.Index(i).String() + if strings.HasPrefix(keyName, prefix) { + keys = append(keys, strings.TrimPrefix(keyName, prefix)) + } + } + return keys, nil +} + +// Length returns the number of keys in externalStorage. +// +// Doc: https://developer.mozilla.org/en-US/docs/Web/API/Storage/length +func (ls *HavenStorageJS) Length() (int, error) { + promise := ls.Call("length") + result, jsErr := utils.Await(promise) + if jsErr != nil { + return 0, js.Error{Value: jsErr[0]} + } + return result[0].Int(), nil +} -- GitLab From 74a3f642067e748141ab3df728543511216c4301 Mon Sep 17 00:00:00 2001 From: thisisommore <ommore501@gmail.com> Date: Thu, 6 Mar 2025 12:31:38 +0000 Subject: [PATCH 2/6] remove unsupported methods (Key, Length), add StorageOperation enum, and rename RemoveItem() to Delete() for KV compliance - Remove unused methods: Key(), Keys(), and Length() - Add StorageOperation enum for consistent method calls - Rename RemoveItem() to Delete() to follow KV interface --- storage/externalStorage.go | 101 +++++++++++++------------------------ 1 file changed, 34 insertions(+), 67 deletions(-) diff --git a/storage/externalStorage.go b/storage/externalStorage.go index 1903616..d9a7897 100644 --- a/storage/externalStorage.go +++ b/storage/externalStorage.go @@ -40,9 +40,9 @@ type ExternalStorage interface { // given key name. Returns an error if external storage quota has been reached. Set(key string, value []byte) error - // RemoveItem removes a key's value from external storage given its name. If + // Delete removes a key's value from external storage given its name. If // there is no item with the given key, this function does nothing. - RemoveItem(keyName string) error + Delete(keyName string) error // Clear clears all the keys in storage. Returns the number of keys cleared and any error. Clear() (int, error) @@ -51,17 +51,9 @@ type ExternalStorage interface { // keys cleared and any error. ClearPrefix(prefix string) (int, error) - // Key returns the name of the nth key in externalStorage. Returns - // os.ErrNotExist if the key does not exist. The order of keys is not - // defined. - Key(n int) (string, error) - // Keys returns a list of all key names in external storage. Keys() ([]string, error) - // Length returns the number of keys in externalStorage. - Length() (int, error) - // ExternalStorageUNSAFE returns the underlying external storage wrapper. This can // be UNSAFE and should only be used if you know what you are doing. // @@ -120,8 +112,8 @@ func (ls *externalStorage) Set(keyName string, keyValue []byte) error { // RemoveItem removes a key's value from external storage given its name. If there // is no item with the given key, this function does nothing. -func (ls *externalStorage) RemoveItem(keyName string) error { - return ls.v.RemoveItem(ls.prefix + keyName) +func (ls *externalStorage) Delete(keyName string) error { + return ls.v.Delete(ls.prefix + keyName) } // Clear clears all the keys in storage. Returns the number of keys cleared and any error. @@ -134,7 +126,7 @@ func (ls *externalStorage) Clear() (int, error) { // Loop through each key for _, keyName := range keys { - if err := ls.RemoveItem(keyName); err != nil { + if err := ls.Delete(keyName); err != nil { return 0, err } } @@ -153,7 +145,7 @@ func (ls *externalStorage) ClearPrefix(prefix string) (int, error) { // Loop through each key for _, keyName := range keys { - if err := ls.RemoveItem(prefix + keyName); err != nil { + if err := ls.Delete(prefix + keyName); err != nil { return 0, err } } @@ -161,16 +153,6 @@ func (ls *externalStorage) ClearPrefix(prefix string) (int, error) { return len(keys), nil } -// Key returns the name of the nth key in externalStorage. Return [os.ErrNotExist] -// if the key does not exist. The order of keys is not defined. -func (ls *externalStorage) Key(n int) (string, error) { - keyName, err := ls.v.Key(n) - if err != nil { - return "", err - } - return strings.TrimPrefix(keyName, ls.prefix), nil -} - // Keys returns a list of all key names in external storage. func (ls *externalStorage) Keys() ([]string, error) { keys, err := ls.v.KeysPrefix(ls.prefix) @@ -180,15 +162,6 @@ func (ls *externalStorage) Keys() ([]string, error) { return keys, nil } -// Length returns the number of keys in externalStorage. -func (ls *externalStorage) Length() (int, error) { - length, err := ls.v.Length() - if err != nil { - return 0, err - } - return length, nil -} - // ExternalStorageUNSAFE returns the underlying external storage wrapper. This can be // UNSAFE and should only be used if you know what you are doing. // @@ -205,19 +178,41 @@ func (ls *externalStorage) ExternalStorageUNSAFE() *HavenStorageJS { // Javascript Wrappers // //////////////////////////////////////////////////////////////////////////////// +// StorageOperation defines the supported operations for HavenStorageJS +type StorageOperation string + +const ( + // GetItemOp represents the "getItem" operation + GetItemOp StorageOperation = "getItem" + // SetItemOp represents the "setItem" operation + SetItemOp StorageOperation = "setItem" + // DeleteOp represents the "delete" operation + DeleteOp StorageOperation = "delete" + // ClearOp represents the "clear" operation + ClearOp StorageOperation = "clear" + // KeysOp represents the "getKeys" operation + KeysOp StorageOperation = "getKeys" +) + // HavenStorageJS stores the Javascript window.havenStorage object and wraps all // of its methods and fields to handle type conversations and errors. type HavenStorageJS struct { js.Value } +// callStorage is a helper function that calls the specified operation on the storage object +// with the provided arguments and returns the result. +func (ls *HavenStorageJS) callStorage(op StorageOperation, args ...interface{}) js.Value { + return ls.Call(string(op), args...) +} + // GetItem returns the value from the external storage given its key name. Returns // [os.ErrNotExist] if the key does not exist. // // Doc: https://developer.mozilla.org/en-US/docs/Web/API/Storage/getItem func (ls *HavenStorageJS) GetItem(keyName string) (keyValue string, err error) { defer exception.Catch(&err) - promise := ls.Call("getItem", keyName) + promise := ls.callStorage(GetItemOp, keyName) result, jsErr := utils.Await(promise) if jsErr != nil { return "", js.Error{Value: jsErr[0]} @@ -234,7 +229,7 @@ func (ls *HavenStorageJS) GetItem(keyName string) (keyValue string, err error) { // Doc: https://developer.mozilla.org/en-US/docs/Web/API/Storage/setItem func (ls *HavenStorageJS) SetItem(keyName, keyValue string) (err error) { defer exception.Catch(&err) - promise := ls.Call("setItem", keyName, keyValue) + promise := ls.callStorage(SetItemOp, keyName, keyValue) _, jsErr := utils.Await(promise) if jsErr != nil { return js.Error{Value: jsErr[0]} @@ -246,8 +241,8 @@ func (ls *HavenStorageJS) SetItem(keyName, keyValue string) (err error) { // is no item with the given key, this function does nothing. // // Doc: https://developer.mozilla.org/en-US/docs/Web/API/Storage/removeItem -func (ls *HavenStorageJS) RemoveItem(keyName string) error { - promise := ls.Call("removeItem", keyName) +func (ls *HavenStorageJS) Delete(keyName string) error { + promise := ls.callStorage(DeleteOp, keyName) _, jsErr := utils.Await(promise) if jsErr != nil { return js.Error{Value: jsErr[0]} @@ -259,7 +254,7 @@ func (ls *HavenStorageJS) RemoveItem(keyName string) error { // // Doc: https://developer.mozilla.org/en-US/docs/Web/API/Storage/clear func (ls *HavenStorageJS) Clear() error { - promise := ls.Call("clear") + promise := ls.callStorage(ClearOp) _, jsErr := utils.Await(promise) if jsErr != nil { return js.Error{Value: jsErr[0]} @@ -267,22 +262,6 @@ func (ls *HavenStorageJS) Clear() error { return nil } -// Key returns the name of the nth key in externalStorage. Return [os.ErrNotExist] -// if the key does not exist. The order of keys is not defined. -// -// Doc: https://developer.mozilla.org/en-US/docs/Web/API/Storage/key -func (ls *HavenStorageJS) Key(n int) (keyName string, err error) { - defer exception.Catch(&err) - promise := ls.Call("key", n) - result, jsErr := utils.Await(promise) - if jsErr != nil { - return "", js.Error{Value: jsErr[0]} - } - if result[0].IsNull() { - return "", os.ErrNotExist - } - return result[0].String(), nil -} // Keys returns a list of all key names in external storage. func (ls *HavenStorageJS) Keys() ([]string, error) { @@ -303,7 +282,7 @@ func (ls *HavenStorageJS) Keys() ([]string, error) { // KeysPrefix returns a list of all key names in external storage with the given // prefix and trims the prefix from each key name. func (ls *HavenStorageJS) KeysPrefix(prefix string) ([]string, error) { - promise := ls.Call("keys") + promise := ls.callStorage(KeysOp) result, jsErr := utils.Await(promise) if jsErr != nil { return []string{}, js.Error{Value: jsErr[0]} @@ -319,15 +298,3 @@ func (ls *HavenStorageJS) KeysPrefix(prefix string) ([]string, error) { } return keys, nil } - -// Length returns the number of keys in externalStorage. -// -// Doc: https://developer.mozilla.org/en-US/docs/Web/API/Storage/length -func (ls *HavenStorageJS) Length() (int, error) { - promise := ls.Call("length") - result, jsErr := utils.Await(promise) - if jsErr != nil { - return 0, js.Error{Value: jsErr[0]} - } - return result[0].Int(), nil -} -- GitLab From 441f03d0f83768ad46d049841ff872d0bac1b415 Mon Sep 17 00:00:00 2001 From: thisisommore <ommore501@gmail.com> Date: Fri, 7 Mar 2025 16:37:16 +0000 Subject: [PATCH 3/6] add back Key method --- storage/externalStorage.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/storage/externalStorage.go b/storage/externalStorage.go index d9a7897..413f208 100644 --- a/storage/externalStorage.go +++ b/storage/externalStorage.go @@ -10,6 +10,7 @@ package storage import ( + "errors" "os" "strings" "syscall/js" @@ -29,6 +30,8 @@ import ( // bytes without any zeros to make them more unique. const externalStorageWasmPrefix = "🞮🞮" +var UnimplementedErr = errors.New("not implemented") + // ExternalStorage defines an interface for setting persistent state in a KV format // specifically for web-based implementations. type ExternalStorage interface { @@ -51,6 +54,11 @@ type ExternalStorage interface { // keys cleared and any error. ClearPrefix(prefix string) (int, error) + // Key returns the name of the nth key in externalStorage. Returns + // os.ErrNotExist if the key does not exist. The order of keys is not + // defined. + Key(n int) (string, error) + // Keys returns a list of all key names in external storage. Keys() ([]string, error) @@ -153,6 +161,16 @@ func (ls *externalStorage) ClearPrefix(prefix string) (int, error) { return len(keys), nil } +// Key returns the name of the nth key in externalStorage. Return [os.ErrNotExist] +// if the key does not exist. The order of keys is not defined. +func (ls *externalStorage) Key(n int) (string, error) { + keyName, err := ls.v.Key(n) + if err != nil { + return "", err + } + return strings.TrimPrefix(keyName, ls.prefix), nil +} + // Keys returns a list of all key names in external storage. func (ls *externalStorage) Keys() ([]string, error) { keys, err := ls.v.KeysPrefix(ls.prefix) @@ -262,6 +280,25 @@ func (ls *HavenStorageJS) Clear() error { return nil } +// Key returns the name of the nth key in externalStorage. Return [os.ErrNotExist] +// if the key does not exist. The order of keys is not defined. +// +// Doc: https://developer.mozilla.org/en-US/docs/Web/API/Storage/key +func (ls *HavenStorageJS) Key(n int) (keyName string, err error) { + defer exception.Catch(&err) + promise := ls.Call("key", n) + result, jsErr := utils.Await(promise) + if jsErr != nil { + if jsErr[0].Type() == js.TypeString && jsErr[0].String() == "not implemented" { + return "", UnimplementedErr + } + return "", js.Error{Value: jsErr[0]} + } + if result[0].IsNull() { + return "", os.ErrNotExist + } + return result[0].String(), nil +} // Keys returns a list of all key names in external storage. func (ls *HavenStorageJS) Keys() ([]string, error) { -- GitLab From def85d4dd0eabd5b8279d5b4b52b36923811d8ee Mon Sep 17 00:00:00 2001 From: thisisommore <ommore501@gmail.com> Date: Sat, 8 Mar 2025 10:12:32 +0000 Subject: [PATCH 4/6] handle unimplemented methods --- storage/externalStorage.go | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/storage/externalStorage.go b/storage/externalStorage.go index 413f208..7b60c44 100644 --- a/storage/externalStorage.go +++ b/storage/externalStorage.go @@ -87,6 +87,17 @@ type externalStorage struct { // jsStorage is the global that stores Javascript as window.havenStorage. var jsExternalStorage ExternalStorage = newExternalStorage(externalStorageWasmPrefix) +// checkUnimplementedErr checks if the error is UnimplementedErr, if yes return +// UnimplementedErr otherwise return the error +func checkUnimplementedErr(jsErr []js.Value) error { + // todo it can be of non error type + jsError := js.Error{Value: jsErr[0]} + if jsError.Error() == "JavaScript error: not implemented" { + return UnimplementedErr + } + return jsError +} + // newExternalStorage creates a new externalStorage object with the specified prefix. func newExternalStorage(prefix string) *externalStorage { return &externalStorage{ @@ -233,7 +244,7 @@ func (ls *HavenStorageJS) GetItem(keyName string) (keyValue string, err error) { promise := ls.callStorage(GetItemOp, keyName) result, jsErr := utils.Await(promise) if jsErr != nil { - return "", js.Error{Value: jsErr[0]} + return "", checkUnimplementedErr(jsErr) } if result[0].IsNull() { return "", os.ErrNotExist @@ -250,7 +261,7 @@ func (ls *HavenStorageJS) SetItem(keyName, keyValue string) (err error) { promise := ls.callStorage(SetItemOp, keyName, keyValue) _, jsErr := utils.Await(promise) if jsErr != nil { - return js.Error{Value: jsErr[0]} + return checkUnimplementedErr(jsErr) } return nil } @@ -263,7 +274,7 @@ func (ls *HavenStorageJS) Delete(keyName string) error { promise := ls.callStorage(DeleteOp, keyName) _, jsErr := utils.Await(promise) if jsErr != nil { - return js.Error{Value: jsErr[0]} + return checkUnimplementedErr(jsErr) } return nil } @@ -275,7 +286,7 @@ func (ls *HavenStorageJS) Clear() error { promise := ls.callStorage(ClearOp) _, jsErr := utils.Await(promise) if jsErr != nil { - return js.Error{Value: jsErr[0]} + return checkUnimplementedErr(jsErr) } return nil } @@ -289,10 +300,7 @@ func (ls *HavenStorageJS) Key(n int) (keyName string, err error) { promise := ls.Call("key", n) result, jsErr := utils.Await(promise) if jsErr != nil { - if jsErr[0].Type() == js.TypeString && jsErr[0].String() == "not implemented" { - return "", UnimplementedErr - } - return "", js.Error{Value: jsErr[0]} + return "", checkUnimplementedErr(jsErr) } if result[0].IsNull() { return "", os.ErrNotExist @@ -305,7 +313,7 @@ func (ls *HavenStorageJS) Keys() ([]string, error) { promise := ls.Call("keys") result, jsErr := utils.Await(promise) if jsErr != nil { - return []string{}, js.Error{Value: jsErr[0]} + return []string{}, checkUnimplementedErr(jsErr) } keysJS := result[0] @@ -322,7 +330,7 @@ func (ls *HavenStorageJS) KeysPrefix(prefix string) ([]string, error) { promise := ls.callStorage(KeysOp) result, jsErr := utils.Await(promise) if jsErr != nil { - return []string{}, js.Error{Value: jsErr[0]} + return []string{}, checkUnimplementedErr(jsErr) } keysJS := result[0] -- GitLab From 3e635b44d91a7a06ac22c00a2bd08b311cbfc958 Mon Sep 17 00:00:00 2001 From: thisisommore <ommore501@gmail.com> Date: Sat, 8 Mar 2025 10:13:53 +0000 Subject: [PATCH 5/6] introduce test file for externalStorage --- storage/externalStorage_test.go | 292 ++++++++++++++++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 storage/externalStorage_test.go diff --git a/storage/externalStorage_test.go b/storage/externalStorage_test.go new file mode 100644 index 0000000..6ec4d4f --- /dev/null +++ b/storage/externalStorage_test.go @@ -0,0 +1,292 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright © 2022 xx foundation // +// // +// Use of this source code is governed by a license that can be found in the // +// LICENSE file. // +//////////////////////////////////////////////////////////////////////////////// + +//go:build js && wasm + +package storage + +import ( + "bytes" + "os" + "reflect" + "strconv" + "syscall/js" + "testing" + + "github.com/pkg/errors" +) + +// setupMockHavenStorage sets up a mock implementation of havenStorage in the JavaScript environment +func setupMockHavenStorage(_ *testing.T) { + const setupScript = ` + const havenStorage = { + getItem: function(key) { + return Promise.resolve(localStorage.getItem(key)); + }, + + setItem: function(key, value) { + localStorage.setItem(key, value); + return Promise.resolve(); + }, + + delete: function(key) { + localStorage.removeItem(key); + return Promise.resolve(); + }, + + clear: function() { + localStorage.clear(); + return Promise.resolve(); + }, + + getKeys: function() { + return Promise.resolve(Object.keys(localStorage)); + }, + + key: function(index) { + // To Test unimplemented error + return Promise.reject(new Error('not implemented')); + } + }; + + // Initialize mock storage + window._mockHavenStorage = {}; + window.havenStorage = havenStorage; + ` + js.Global().Call("eval", setupScript) + jsExternalStorage = newExternalStorage(externalStorageWasmPrefix) +} + +// Unit test of GetExternalStorage. +func TestGetExternalStorage(t *testing.T) { + setupMockHavenStorage(t) + + expected := &externalStorage{ + v: &HavenStorageJS{js.Global().Get("havenStorage")}, + prefix: externalStorageWasmPrefix, + } + + es := GetExternalStorage() + + if !reflect.DeepEqual(expected, es) { + t.Errorf("Did not receive expected externalStorage."+ + "\nexpected: %+v\nreceived: %+v", expected, es) + } +} + +// Tests that a value set with externalStorage.Set and retrieved with +// externalStorage.Get matches the original. +func TestExternalStorage_Get_Set(t *testing.T) { + setupMockHavenStorage(t) + + values := map[string][]byte{ + "key1": []byte("key value"), + "key2": {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + "key3": {0, 49, 0, 0, 0, 38, 249, 93, 242, 189, 222, 32, 138, 248, 121, + 151, 42, 108, 82, 199, 163, 61, 4, 200, 140, 231, 225, 20, 35, 243, + 253, 161, 61, 2, 227, 208, 173, 183, 33, 66, 236, 107, 105, 119, 26, + 42, 44, 60, 109, 172, 38, 47, 220, 17, 129, 4, 234, 241, 141, 81, + 84, 185, 32, 120, 115, 151, 128, 196, 143, 117, 222, 78, 44, 115, + 109, 20, 249, 46, 158, 139, 231, 157, 54, 219, 141, 252}, + } + + for keyName, keyValue := range values { + err := jsExternalStorage.Set(keyName, keyValue) + if err != nil { + t.Errorf("Failed to set %q: %+v", keyName, err) + } + + loadedValue, err := jsExternalStorage.Get(keyName) + if err != nil { + t.Errorf("Failed to load %q: %+v", keyName, err) + } else if !bytes.Equal(keyValue, loadedValue) { + t.Errorf("Loaded value does not match original for %q"+ + "\nexpected: %q\nreceived: %q", keyName, keyValue, loadedValue) + } + } +} + +// Tests that externalStorage.Delete deletes a key from the store and that it +// cannot be retrieved. +func TestExternalStorage_Delete(t *testing.T) { + setupMockHavenStorage(t) + + keyName := "key" + if err := jsExternalStorage.Set(keyName, []byte("value")); err != nil { + t.Errorf("Failed to set %q: %+v", keyName, err) + } + + err := jsExternalStorage.Delete(keyName) + if err != nil { + t.Errorf("Failed to delete key %q: %+v", keyName, err) + } + + _, err = jsExternalStorage.Get(keyName) + if err == nil || !errors.Is(err, os.ErrNotExist) { + t.Errorf("Failed to remove %q: %+v", keyName, err) + } +} + +// Tests that externalStorage.Clear deletes all the WASM keys from storage and +// does not remove any others +func TestExternalStorage_Clear(t *testing.T) { + setupMockHavenStorage(t) + + // Clear any existing data + jsExternalStorage.ExternalStorageUNSAFE().Clear() + + const numKeys = 10 + var yesPrefix, noPrefix []string + + for i := 0; i < numKeys; i++ { + keyName := "keyNum" + strconv.Itoa(i) + if i%2 == 0 { + yesPrefix = append(yesPrefix, keyName) + err := jsExternalStorage.Set(keyName, []byte(strconv.Itoa(i))) + if err != nil { + t.Errorf("Failed to set with prefix %q: %+v", keyName, err) + } + } else { + noPrefix = append(noPrefix, keyName) + err := jsExternalStorage.ExternalStorageUNSAFE().SetItem(keyName, strconv.Itoa(i)) + if err != nil { + t.Errorf("Failed to set with no prefix %q: %+v", keyName, err) + } + } + } + + n, err := jsExternalStorage.Clear() + if err != nil { + t.Errorf("Failed to clear storage: %+v", err) + } + if n != numKeys/2 { + t.Errorf("Incorrect number of keys.\nexpected: %d\nreceived: %d", + numKeys/2, n) + } + + for _, keyName := range noPrefix { + if _, err := jsExternalStorage.ExternalStorageUNSAFE().GetItem(keyName); err != nil { + t.Errorf("Could not get keyName %q: %+v", keyName, err) + } + } + for _, keyName := range yesPrefix { + keyValue, err := jsExternalStorage.Get(keyName) + if err == nil || !errors.Is(err, os.ErrNotExist) { + t.Errorf("Found keyName %q: %q", keyName, keyValue) + } + } +} + +// Tests that externalStorage.ClearPrefix deletes only the keys with the given +// prefix. +func TestExternalStorage_ClearPrefix(t *testing.T) { + setupMockHavenStorage(t) + + // Clear any existing data + jsExternalStorage.ExternalStorageUNSAFE().Clear() + + const numKeys = 10 + var yesPrefix, noPrefix []string + prefix := "keyNamePrefix/" + + for i := 0; i < numKeys; i++ { + keyName := "keyNum " + strconv.Itoa(i) + if i%2 == 0 { + keyName = prefix + keyName + yesPrefix = append(yesPrefix, keyName) + } else { + noPrefix = append(noPrefix, keyName) + } + + if err := jsExternalStorage.Set(keyName, []byte(strconv.Itoa(i))); err != nil { + t.Errorf("Failed to set %q: %+v", keyName, err) + } + } + + n, err := jsExternalStorage.ClearPrefix(prefix) + if err != nil { + t.Errorf("Failed to clear prefix: %+v", err) + } + if n != numKeys/2 { + t.Errorf("Incorrect number of keys.\nexpected: %d\nreceived: %d", + numKeys/2, n) + } + + for _, keyName := range noPrefix { + if _, err := jsExternalStorage.Get(keyName); err != nil { + t.Errorf("Could not get keyName %q: %+v", keyName, err) + } + } + for _, keyName := range yesPrefix { + keyValue, err := jsExternalStorage.Get(keyName) + if err == nil || !errors.Is(err, os.ErrNotExist) { + t.Errorf("Found keyName %q: %q", keyName, keyValue) + } + } +} + +// Tests that externalStorage.Key return all added keys when looping through all +// indexes. +func TestExternalStorage_Key(t *testing.T) { + setupMockHavenStorage(t) + + // This test will verify that the UnimplementedErr is properly returned + // since the mock implementation rejects the key method + _, err := jsExternalStorage.Key(0) + if err == nil || !errors.Is(err, UnimplementedErr) { + t.Errorf("Expected UnimplementedErr for Key method.\nexpected: %v\nreceived: %v", + UnimplementedErr, err) + } +} + +// Tests that externalStorage.Get returns the error os.ErrNotExist when the key +// does not exist in storage. +func TestExternalStorage_Get_NotExistError(t *testing.T) { + setupMockHavenStorage(t) + + _, err := jsExternalStorage.Get("someKey") + if err == nil || !errors.Is(err, os.ErrNotExist) { + t.Errorf("Incorrect error for non existant key."+ + "\nexpected: %v\nreceived: %v", os.ErrNotExist, err) + } +} + +// Tests that externalStorage.Keys return a list that contains all the added keys. +func TestExternalStorage_Keys(t *testing.T) { + setupMockHavenStorage(t) + + // Clear any existing data + jsExternalStorage.ExternalStorageUNSAFE().Clear() + + values := map[string][]byte{ + "key1": []byte("key value"), + "key2": {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + "key3": {0, 49, 0, 0, 0, 38, 249, 93}, + } + + for keyName, keyValue := range values { + if err := jsExternalStorage.Set(keyName, keyValue); err != nil { + t.Errorf("Failed to set %q: %+v", keyName, err) + } + } + + keys, err := jsExternalStorage.Keys() + if err != nil { + t.Errorf("Failed to get keys: %+v", err) + } + + if len(keys) != len(values) { + t.Errorf("Incorrect number of keys.\nexpected: %d\nreceived: %d", + len(values), len(keys)) + } + + for i, keyName := range keys { + if _, exists := values[keyName]; !exists { + t.Errorf("Key %q does not exist (%d).", keyName, i) + } + } +} -- GitLab From e94d97011aa9e0bf4e183b5c8c9c8bc8d5fb1abe Mon Sep 17 00:00:00 2001 From: thisisommore <ommore501@gmail.com> Date: Sun, 9 Mar 2025 05:56:37 +0000 Subject: [PATCH 6/6] add support for GetPrefix HasPrefix Prefix Root IsMemStore Length --- storage/externalStorage.go | 62 ++++++++++++++++++++++- storage/externalStorage_test.go | 89 ++++++++++++++++++++++++++++++++- 2 files changed, 149 insertions(+), 2 deletions(-) diff --git a/storage/externalStorage.go b/storage/externalStorage.go index 7b60c44..3f41d0b 100644 --- a/storage/externalStorage.go +++ b/storage/externalStorage.go @@ -62,6 +62,24 @@ type ExternalStorage interface { // Keys returns a list of all key names in external storage. Keys() ([]string, error) + // GetPrefix returns the full Prefix of the KV + GetPrefix() string + + // HasPrefix returns whether this prefix exists in the KV + HasPrefix(prefix string) bool + + // Prefix returns a new KV with the new prefix appending + Prefix(prefix string) (ExternalStorage, error) + + // Root returns the KV with no prefixes + Root() ExternalStorage + + // IsMemStore returns true if the underlying KV is memory based + IsMemStore() (bool, error) + + // Length returns the number of keys in localStorage. + Length() int + // ExternalStorageUNSAFE returns the underlying external storage wrapper. This can // be UNSAFE and should only be used if you know what you are doing. // @@ -191,6 +209,30 @@ func (ls *externalStorage) Keys() ([]string, error) { return keys, nil } +func (ls *externalStorage) GetPrefix() string { + return ls.prefix +} + +func (ls *externalStorage) HasPrefix(prefix string) bool { + return strings.HasPrefix(ls.prefix, prefix) +} + +func (ls *externalStorage) Prefix(prefix string) (ExternalStorage, error) { + return newExternalStorage(ls.prefix + prefix), nil +} + +func (ls *externalStorage) Root() ExternalStorage { + return newExternalStorage("") +} + +func (ls *externalStorage) IsMemStore() (bool, error) { + return ls.v.IsMemStore() +} + +func (ls *externalStorage) Length() int { + return ls.v.Length() +} + // ExternalStorageUNSAFE returns the underlying external storage wrapper. This can be // UNSAFE and should only be used if you know what you are doing. // @@ -221,6 +263,9 @@ const ( ClearOp StorageOperation = "clear" // KeysOp represents the "getKeys" operation KeysOp StorageOperation = "getKeys" + + // IsMemStoreOp represents the "isMemStore" operation + IsMemStoreOp StorageOperation = "isMemStore" ) // HavenStorageJS stores the Javascript window.havenStorage object and wraps all @@ -310,7 +355,7 @@ func (ls *HavenStorageJS) Key(n int) (keyName string, err error) { // Keys returns a list of all key names in external storage. func (ls *HavenStorageJS) Keys() ([]string, error) { - promise := ls.Call("keys") + promise := ls.callStorage(KeysOp) result, jsErr := utils.Await(promise) if jsErr != nil { return []string{}, checkUnimplementedErr(jsErr) @@ -343,3 +388,18 @@ func (ls *HavenStorageJS) KeysPrefix(prefix string) ([]string, error) { } return keys, nil } + +func (ls *HavenStorageJS) IsMemStore() (bool, error) { + result := ls.callStorage(IsMemStoreOp) + var err error + defer exception.Catch(&err) + if err != nil { + //TODO + jsErr, ok := err.(js.Error) + if ok { + return false, checkUnimplementedErr([]js.Value{js.ValueOf(jsErr)}) + } + return false, err + } + return result.Bool(), nil +} diff --git a/storage/externalStorage_test.go b/storage/externalStorage_test.go index 6ec4d4f..418d2e0 100644 --- a/storage/externalStorage_test.go +++ b/storage/externalStorage_test.go @@ -50,7 +50,15 @@ func setupMockHavenStorage(_ *testing.T) { key: function(index) { // To Test unimplemented error return Promise.reject(new Error('not implemented')); - } + }, + + isMemStore: function() { + return Promise.resolve(false); + }, + + length: function() { + return Promise.resolve(0); + }, }; // Initialize mock storage @@ -61,6 +69,85 @@ func setupMockHavenStorage(_ *testing.T) { jsExternalStorage = newExternalStorage(externalStorageWasmPrefix) } +func TestExternalStorage_GetPrefix(t *testing.T) { + setupMockHavenStorage(t) + + es := GetExternalStorage() + + prefix := es.GetPrefix() + if prefix != externalStorageWasmPrefix { + t.Errorf("Expected prefix to be %q, got %q", externalStorageWasmPrefix, prefix) + } +} + +func TestExternalStorage_HasPrefix(t *testing.T) { + setupMockHavenStorage(t) + + es := GetExternalStorage() + + hasPrefix := es.HasPrefix(externalStorageWasmPrefix) + if !hasPrefix { + t.Errorf("Expected hasPrefix to be true") + } + + hasPrefix = es.HasPrefix("someOtherPrefix") + if hasPrefix { + t.Errorf("Expected hasPrefix to be false") + } +} + +func TestExternalStorage_Prefix(t *testing.T) { + setupMockHavenStorage(t) + + es := GetExternalStorage() + + prefix, err := es.Prefix("testPrefix") + if err != nil { + t.Errorf("Failed to get prefix: %+v", err) + } + + if prefix.GetPrefix() != externalStorageWasmPrefix+"testPrefix" { + t.Errorf("Expected prefix to be %q, got %q", externalStorageWasmPrefix+"testPrefix", prefix.GetPrefix()) + } +} + +func TestExternalStorage_Root(t *testing.T) { + setupMockHavenStorage(t) + + es := GetExternalStorage() + + root := es.Root() + if root.GetPrefix() != "" { + t.Errorf("Expected prefix to be %q, got %q", "", root.GetPrefix()) + } +} + +func TestExternalStorage_IsMemStore(t *testing.T) { + setupMockHavenStorage(t) + + es := GetExternalStorage() + + isMemStore, err := es.IsMemStore() + if err != nil { + t.Errorf("Failed to get isMemStore: %+v", err) + } + + if isMemStore { + t.Errorf("Expected memstore to be false") + } +} + +func TestExternalStorage_Length(t *testing.T) { + setupMockHavenStorage(t) + + es := GetExternalStorage() + + length := es.Length() + if length != 0 { + t.Errorf("Expected length to be %d, got %d", 0, length) + } +} + // Unit test of GetExternalStorage. func TestGetExternalStorage(t *testing.T) { setupMockHavenStorage(t) -- GitLab