added states table for large client storage

.gitlab-ci.yml

@@ -69,9 +69,11 @@ build-workers:
     - GOOS=js GOARCH=wasm go build -ldflags '-w -s' -trimpath -o release/xxdk-channelsIndexedDkWorker.wasm ./indexedDb/impl/channels/...
     - GOOS=js GOARCH=wasm go build -ldflags '-w -s' -trimpath -o release/xxdk-dmIndexedDkWorker.wasm ./indexedDb/impl/dm/...
     - GOOS=js GOARCH=wasm go build -ldflags '-w -s' -trimpath -o release/xxdk-logFileWorker.wasm ./logging/workerThread/...
+    - GOOS=js GOARCH=wasm go build -ldflags '-w -s' -trimpath -o release/xxdk-stateIndexedDbWorker.wasm ./indexedDb/impl/state/...
     - cp indexedDb/impl/channels/channelsIndexedDbWorker.js release/
     - cp indexedDb/impl/dm/dmIndexedDbWorker.js release/
     - cp logging/workerThread/logFileWorker.js release/
+    - cp indexedDb/impl/state/stateIndexedDbWorker.js release/
   artifacts:
     paths:
       - release/
@@ -112,9 +114,11 @@ combine-artifacts:
     - 'curl --fail --location --header "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" --output release/xxdk-channelsIndexedDkWorker.wasm $CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/jobs/$BUILD_WORKERS_JOB_ID/artifacts/release/xxdk-channelsIndexedDkWorker.wasm'
     - 'curl --fail --location --header "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" --output release/xxdk-dmIndexedDkWorker.wasm $CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/jobs/$BUILD_WORKERS_JOB_ID/artifacts/release/xxdk-dmIndexedDkWorker.wasm'
     - 'curl --fail --location --header "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" --output release/xxdk-logFileWorker.wasm $CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/jobs/$BUILD_WORKERS_JOB_ID/artifacts/release/xxdk-logFileWorker.wasm'
+    - 'curl --fail --location --header "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" --output release/xxdk-stateIndexedDbWorker.wasm $CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/jobs/$BUILD_WORKERS_JOB_ID/artifacts/release/xxdk-stateIndexedDbWorker.wasm'
     - 'curl --fail --location --header "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" --output release/channelsIndexedDbWorker.js $CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/jobs/$BUILD_WORKERS_JOB_ID/artifacts/release/channelsIndexedDbWorker.js'
     - 'curl --fail --location --header "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" --output release/dmIndexedDbWorker.js $CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/jobs/$BUILD_WORKERS_JOB_ID/artifacts/release/dmIndexedDbWorker.js'
     - 'curl --fail --location --header "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" --output release/logFileWorker.js $CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/jobs/$BUILD_WORKERS_JOB_ID/artifacts/release/logFileWorker.js'
+    - 'curl --fail --location --header "PRIVATE-TOKEN: $GITLAB_ACCESS_TOKEN" --output release/stateIndexedDbWorker.js $CI_SERVER_URL/api/v4/projects/$CI_PROJECT_ID/jobs/$BUILD_WORKERS_JOB_ID/artifacts/release/stateIndexedDbWorker.js'
     - ls release
   artifacts:
     paths:

Makefile

@@ -32,6 +32,7 @@ binary:
 worker_binaries:
 	GOOS=js GOARCH=wasm go build -ldflags '-w -s' -trimpath -o xxdk-channelsIndexedDkWorker.wasm ./indexedDb/impl/channels/...
 	GOOS=js GOARCH=wasm go build -ldflags '-w -s' -trimpath -o xxdk-dmIndexedDkWorker.wasm ./indexedDb/impl/dm/...
+	GOOS=js GOARCH=wasm go build -ldflags '-w -s' -trimpath -o xxdk-stateIndexedDkWorker.wasm ./indexedDb/impl/state/...
 	GOOS=js GOARCH=wasm go build -ldflags '-w -s' -trimpath -o xxdk-logFileWorker.wasm ./logging/workerThread/...
 
 binaries: binary worker_binaries

indexedDb/impl/channels/implementation.go

@@ -32,9 +32,7 @@ import (
 	"gitlab.com/xx_network/primitives/id"
 )
 
-// wasmModel implements [channels.EventModel] interface, which uses the channels
-// system passed an object that adheres to in order to get events on the
-// channel.
+// wasmModel implements [channels.EventModel] interface backed by IndexedDb.
 // NOTE: This model is NOT thread safe - it is the responsibility of the
 // caller to ensure that its methods are called sequentially.
 type wasmModel struct {

indexedDb/impl/channels/init.go

@@ -43,12 +43,13 @@ func newWASMModel(databaseName string, encryption cryptoChannel.Cipher,
 	openRequest, err := idb.Global().Open(ctx, databaseName, currentVersion,
 		func(db *idb.Database, oldVersion, newVersion uint) error {
 			if oldVersion == newVersion {
-				jww.INFO.Printf("IndexDb version is current: v%d", newVersion)
+				jww.INFO.Printf("IndexDb version for %s is current: v%d",
+					databaseName, newVersion)
 				return nil
 			}
 
-			jww.INFO.Printf("IndexDb upgrade required: v%d -> v%d",
-				oldVersion, newVersion)
+			jww.INFO.Printf("IndexDb upgrade required for %s: v%d -> v%d",
+				databaseName, oldVersion, newVersion)
 
 			if oldVersion == 0 && newVersion >= 1 {
 				err := v1Upgrade(db)

indexedDb/impl/dm/init.go

@@ -49,12 +49,13 @@ func newWASMModel(databaseName string, encryption cryptoChannel.Cipher,
 	openRequest, err := idb.Global().Open(ctx, databaseName, currentVersion,
 		func(db *idb.Database, oldVersion, newVersion uint) error {
 			if oldVersion == newVersion {
-				jww.INFO.Printf("IndexDb version is current: v%d", newVersion)
+				jww.INFO.Printf("IndexDb version for %s is current: v%d",
+					databaseName, newVersion)
 				return nil
 			}
 
-			jww.INFO.Printf("IndexDb upgrade required: v%d -> v%d",
-				oldVersion, newVersion)
+			jww.INFO.Printf("IndexDb upgrade required for %s: v%d -> v%d",
+				databaseName, oldVersion, newVersion)
 
 			if oldVersion == 0 && newVersion >= 1 {
 				err := v1Upgrade(db)

indexedDb/impl/state/callbacks.go

+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2022 xx foundation                                             //
+//                                                                            //
+// Use of this source code is governed by a license that can be found in the  //
+// LICENSE file.                                                              //
+////////////////////////////////////////////////////////////////////////////////
+
+//go:build js && wasm
+
+package main
+
+import (
+	"encoding/json"
+	"github.com/pkg/errors"
+
+	"gitlab.com/elixxir/client/v4/storage/utility"
+	stateWorker "gitlab.com/elixxir/xxdk-wasm/indexedDb/worker/state"
+	"gitlab.com/elixxir/xxdk-wasm/worker"
+)
+
+// manager handles the message callbacks, which is used to
+// send information between the model and the main thread.
+type manager struct {
+	wtm   *worker.ThreadManager
+	model utility.WebState
+}
+
+// registerCallbacks registers all the reception callbacks to manage messages
+// from the main thread.
+func (m *manager) registerCallbacks() {
+	m.wtm.RegisterCallback(stateWorker.NewStateTag, m.newStateCB)
+	m.wtm.RegisterCallback(stateWorker.SetTag, m.setCB)
+	m.wtm.RegisterCallback(stateWorker.GetTag, m.getCB)
+}
+
+// newStateCB is the callback for NewState. Returns an empty
+// slice on success or an error message on failure.
+func (m *manager) newStateCB(data []byte) ([]byte, error) {
+	var msg stateWorker.NewStateMessage
+	err := json.Unmarshal(data, &msg)
+	if err != nil {
+		return []byte{}, errors.Errorf(
+			"failed to JSON unmarshal %T from main thread: %+v", msg, err)
+	}
+
+	m.model, err = NewState(msg.DatabaseName)
+	if err != nil {
+		return []byte(err.Error()), nil
+	}
+
+	return []byte{}, nil
+}
+
+// setCB is the callback for stateModel.Set.
+// Returns nil on error or the resulting byte data on success.
+func (m *manager) setCB(data []byte) ([]byte, error) {
+	var msg stateWorker.TransferMessage
+	err := json.Unmarshal(data, &msg)
+	if err != nil {
+		return nil, errors.Errorf(
+			"failed to JSON unmarshal %T from main thread: %+v", msg, err)
+	}
+
+	return nil, m.model.Set(msg.Key, msg.Value)
+}
+
+// getCB is the callback for stateModel.Get.
+// Returns nil on error or the resulting byte data on success.
+func (m *manager) getCB(data []byte) ([]byte, error) {
+	key := string(data)
+	result, err := m.model.Get(key)
+	msg := stateWorker.TransferMessage{
+		Key:   key,
+		Value: result,
+		Error: err.Error(),
+	}
+
+	return json.Marshal(msg)
+}

indexedDb/impl/state/implementation.go

+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2022 xx foundation                                             //
+//                                                                            //
+// Use of this source code is governed by a license that can be found in the  //
+// LICENSE file.                                                              //
+////////////////////////////////////////////////////////////////////////////////
+
+//go:build js && wasm
+
+package main
+
+import (
+	"encoding/json"
+	"github.com/hack-pad/go-indexeddb/idb"
+	"github.com/pkg/errors"
+	"gitlab.com/elixxir/wasm-utils/utils"
+	"gitlab.com/elixxir/xxdk-wasm/indexedDb/impl"
+	"syscall/js"
+)
+
+// stateModel implements [ClientState] interface backed by IndexedDb.
+// NOTE: This model is NOT thread safe - it is the responsibility of the
+// caller to ensure that its methods are called sequentially.
+type stateModel struct {
+	db *idb.Database
+}
+
+func (s *stateModel) Get(key string) ([]byte, error) {
+	result, err := impl.Get(s.db, stateStoreName, js.ValueOf(key))
+	if err != nil {
+		return nil, err
+	}
+
+	stateObj := &State{}
+	err = json.Unmarshal([]byte(utils.JsToJson(result)), stateObj)
+	if err != nil {
+		return nil, err
+	}
+
+	return stateObj.Value, err
+}
+
+func (s *stateModel) Set(key string, value []byte) error {
+	state := &State{
+		Id:    key,
+		Value: value,
+	}
+
+	// Convert to jsObject
+	newStateJSON, err := json.Marshal(state)
+	if err != nil {
+		return errors.Errorf("Unable to marshal State: %+v", err)
+	}
+	stateObj, err := utils.JsonToJS(newStateJSON)
+	if err != nil {
+		return errors.Errorf("Unable to marshal State: %+v", err)
+	}
+
+	// Store State to database
+	_, err = impl.Put(s.db, stateStoreName, stateObj)
+	if err != nil {
+		return errors.Errorf("Unable to put State: %+v\n%s",
+			err, newStateJSON)
+	}
+	return nil
+}

indexedDb/impl/state/init.go

+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2022 xx foundation                                             //
+//                                                                            //
+// Use of this source code is governed by a license that can be found in the  //
+// LICENSE file.                                                              //
+////////////////////////////////////////////////////////////////////////////////
+
+//go:build js && wasm
+
+package main
+
+import (
+	"github.com/hack-pad/go-indexeddb/idb"
+	jww "github.com/spf13/jwalterweatherman"
+	"gitlab.com/elixxir/client/v4/storage/utility"
+	"gitlab.com/elixxir/xxdk-wasm/indexedDb/impl"
+	"syscall/js"
+)
+
+// currentVersion is the current version of the IndexedDb runtime. Used for
+// migration purposes.
+const currentVersion uint = 1
+
+// NewState returns a [utility.WebState] backed by IndexedDb.
+// The name should be a base64 encoding of the users public key.
+func NewState(databaseName string) (utility.WebState, error) {
+	return newState(databaseName)
+}
+
+// newState creates the given [idb.Database] and returns a stateModel.
+func newState(databaseName string) (*stateModel, error) {
+	// Attempt to open database object
+	ctx, cancel := impl.NewContext()
+	defer cancel()
+	openRequest, err := idb.Global().Open(ctx, databaseName, currentVersion,
+		func(db *idb.Database, oldVersion, newVersion uint) error {
+			if oldVersion == newVersion {
+				jww.INFO.Printf("IndexDb version for %s is current: v%d",
+					databaseName, newVersion)
+				return nil
+			}
+
+			jww.INFO.Printf("IndexDb upgrade required for %s: v%d -> v%d",
+				databaseName, oldVersion, newVersion)
+
+			if oldVersion == 0 && newVersion >= 1 {
+				err := v1Upgrade(db)
+				if err != nil {
+					return err
+				}
+				oldVersion = 1
+			}
+
+			// if oldVersion == 1 && newVersion >= 2 { v2Upgrade(), oldVersion = 2 }
+			return nil
+		})
+	if err != nil {
+		return nil, err
+	}
+
+	// Wait for database open to finish
+	db, err := openRequest.Await(ctx)
+	if err != nil {
+		return nil, err
+	} else if ctx.Err() != nil {
+		return nil, ctx.Err()
+	}
+
+	wrapper := &stateModel{db: db}
+	return wrapper, nil
+}
+
+// v1Upgrade performs the v0 -> v1 database upgrade.
+//
+// This can never be changed without permanently breaking backwards
+// compatibility.
+func v1Upgrade(db *idb.Database) error {
+	storeOpts := idb.ObjectStoreOptions{
+		KeyPath:       js.ValueOf(pkeyName),
+		AutoIncrement: false,
+	}
+	_, err := db.CreateObjectStore(stateStoreName, storeOpts)
+	return err
+}

indexedDb/impl/state/main.go

+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2022 xx foundation                                             //
+//                                                                            //
+// Use of this source code is governed by a license that can be found in the  //
+// LICENSE file.                                                              //
+////////////////////////////////////////////////////////////////////////////////
+
+//go:build js && wasm
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"syscall/js"
+
+	"github.com/spf13/cobra"
+	jww "github.com/spf13/jwalterweatherman"
+
+	"gitlab.com/elixxir/xxdk-wasm/logging"
+	"gitlab.com/elixxir/xxdk-wasm/worker"
+)
+
+// SEMVER is the current semantic version of the xxDK web worker.
+const SEMVER = "0.1.0"
+
+func main() {
+	// Set to os.Args because the default is os.Args[1:] and in WASM, args start
+	// at 0, not 1.
+	channelsCmd.SetArgs(os.Args)
+
+	err := channelsCmd.Execute()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+var channelsCmd = &cobra.Command{
+	Use:     "stateIndexedDbWorker",
+	Short:   "IndexedDb database for state.",
+	Example: "const go = new Go();\ngo.argv = [\"--logLevel=1\"]",
+	Run: func(cmd *cobra.Command, args []string) {
+		// Start logger first to capture all logging events
+		err := logging.EnableLogging(logLevel, -1, 0, "", "")
+		if err != nil {
+			fmt.Printf("Failed to intialize logging: %+v", err)
+			os.Exit(1)
+		}
+
+		jww.INFO.Printf("xxDK state web worker version: v%s", SEMVER)
+
+		jww.INFO.Print("[WW] Starting xxDK WebAssembly State Database Worker.")
+		m := &manager{
+			wtm: worker.NewThreadManager("StateIndexedDbWorker", true),
+		}
+		m.registerCallbacks()
+		m.wtm.SignalReady()
+
+		// Indicate to the Javascript caller that the WASM is ready by resolving
+		// a promise created by the caller.
+		js.Global().Get("onWasmInitialized").Invoke()
+
+		<-make(chan bool)
+		fmt.Println("[WW] Closing xxDK WebAssembly State Database Worker.")
+		os.Exit(0)
+	},
+}
+
+var (
+	logLevel jww.Threshold
+)
+
+func init() {
+	// Initialize all startup flags
+	channelsCmd.Flags().IntVarP((*int)(&logLevel), "logLevel", "l", 2,
+		"Sets the log level output when outputting to the Javascript console. "+
+			"0 = TRACE, 1 = DEBUG, 2 = INFO, 3 = WARN, 4 = ERROR, "+
+			"5 = CRITICAL, 6 = FATAL, -1 = disabled.")
+}

indexedDb/impl/state/model.go

+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2022 xx foundation                                             //
+//                                                                            //
+// Use of this source code is governed by a license that can be found in the  //
+// LICENSE file.                                                              //
+////////////////////////////////////////////////////////////////////////////////
+
+//go:build js && wasm
+
+package main
+
+const (
+	// Text representation of primary key value (keyPath).
+	pkeyName = "id"
+
+	// Text representation of the names of the various [idb.ObjectStore].
+	stateStoreName = "states"
+)
+
+// State defines the IndexedDb representation of a single KV data store.
+type State struct {
+	// Id is a unique identifier for a given State.
+	Id string `json:"id"` // Matches pkeyName
+
+	// Value stores the data contents of the State.
+	Value []byte `json:"value"`
+}

indexedDb/impl/state/stateIndexedDbWorker.js

+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2022 xx foundation                                             //
+//                                                                            //
+// Use of this source code is governed by a license that can be found in the  //
+// LICENSE file.                                                              //
+////////////////////////////////////////////////////////////////////////////////
+
+importScripts('wasm_exec.js');
+
+const isReady = new Promise((resolve) => {
+    self.onWasmInitialized = resolve;
+});
+
+const go = new Go();
+const binPath = 'xxdk-stateIndexedDkWorker.wasm'
+WebAssembly.instantiateStreaming(fetch(binPath), go.importObject).then(async (result) => {
+    go.run(result.instance);
+    await isReady;
+}).catch((err) => {
+    console.error(err);
+});
\ No newline at end of file

indexedDb/worker/state/implementation.go

+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2022 xx foundation                                             //
+//                                                                            //
+// Use of this source code is governed by a license that can be found in the  //
+// LICENSE file.                                                              //
+////////////////////////////////////////////////////////////////////////////////
+
+//go:build js && wasm
+
+package dm
+
+import (
+	"encoding/json"
+	"github.com/pkg/errors"
+	"time"
+
+	"gitlab.com/elixxir/xxdk-wasm/worker"
+)
+
+type wasmModel struct {
+	wh *worker.Manager
+}
+
+// TransferMessage is JSON marshalled and sent to the worker.
+type TransferMessage struct {
+	Key   string `json:"key"`
+	Value []byte `json:"value"`
+	Error string `json:"error"`
+}
+
+func (w *wasmModel) Set(key string, value []byte) error {
+	msg := TransferMessage{
+		Key:   key,
+		Value: value,
+	}
+
+	data, err := json.Marshal(msg)
+	if err != nil {
+		return errors.Errorf(
+			"Could not JSON marshal payload for TransferMessage: %+v", err)
+	}
+
+	resultChan := make(chan []byte)
+	w.wh.SendMessage(SetTag, data,
+		func(data []byte) {
+			resultChan <- data
+		})
+
+	select {
+	case result := <-resultChan:
+		return errors.New(string(result))
+	case <-time.After(worker.ResponseTimeout):
+		return errors.Errorf("Timed out after %s waiting for response from the "+
+			"worker about Get", worker.ResponseTimeout)
+	}
+}
+
+func (w *wasmModel) Get(key string) ([]byte, error) {
+	resultChan := make(chan []byte)
+	w.wh.SendMessage(GetTag, []byte(key),
+		func(data []byte) {
+			resultChan <- data
+		})
+
+	select {
+	case result := <-resultChan:
+		var msg TransferMessage
+		err := json.Unmarshal(result, &msg)
+		if err != nil {
+			return nil, errors.Errorf(
+				"failed to JSON unmarshal %T from main thread: %+v", msg, err)
+		}
+
+		if len(msg.Error) > 0 {
+			return nil, errors.New(msg.Error)
+		}
+		return msg.Value, nil
+	case <-time.After(worker.ResponseTimeout):
+		return nil, errors.Errorf("Timed out after %s waiting for response from the "+
+			"worker about Get", worker.ResponseTimeout)
+	}
+}

indexedDb/worker/state/init.go

+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2022 xx foundation                                             //
+//                                                                            //
+// Use of this source code is governed by a license that can be found in the  //
+// LICENSE file.                                                              //
+////////////////////////////////////////////////////////////////////////////////
+
+//go:build js && wasm
+
+package dm
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"gitlab.com/elixxir/client/v4/storage/utility"
+	"gitlab.com/elixxir/xxdk-wasm/storage"
+	"gitlab.com/elixxir/xxdk-wasm/worker"
+)
+
+// databaseSuffix is the suffix to be appended to the name of the database.
+const databaseSuffix = "_speakeasy_state"
+
+// NewStateMessage is JSON marshalled and sent to the worker for
+// [NewState].
+type NewStateMessage struct {
+	DatabaseName string `json:"databaseName"`
+}
+
+// NewState returns a [utility.WebState] backed by indexeddb.
+// The name should be a base64 encoding of the users public key.
+func NewState(path, wasmJsPath string) (utility.WebState, error) {
+	databaseName := path + databaseSuffix
+
+	wh, err := worker.NewManager(wasmJsPath, "stateIndexedDb", true)
+	if err != nil {
+		return nil, err
+	}
+
+	// Store the database name
+	err = storage.StoreIndexedDb(databaseName)
+	if err != nil {
+		return nil, err
+	}
+
+	msg := NewStateMessage{
+		DatabaseName: databaseName,
+	}
+
+	payload, err := json.Marshal(msg)
+	if err != nil {
+		return nil, err
+	}
+
+	dataChan := make(chan []byte)
+	wh.SendMessage(NewStateTag, payload,
+		func(data []byte) { dataChan <- data })
+
+	select {
+	case data := <-dataChan:
+		if len(data) > 0 {
+			return nil, errors.New(string(data))
+		}
+	case <-time.After(worker.ResponseTimeout):
+		return nil, errors.Errorf("timed out after %s waiting for indexedDB "+
+			"database in worker to initialize", worker.ResponseTimeout)
+	}
+
+	return &wasmModel{wh}, nil
+}

indexedDb/worker/state/tags.go

+////////////////////////////////////////////////////////////////////////////////
+// Copyright © 2022 xx foundation                                             //
+//                                                                            //
+// Use of this source code is governed by a license that can be found in the  //
+// LICENSE file.                                                              //
+////////////////////////////////////////////////////////////////////////////////
+
+//go:build js && wasm
+
+package dm
+
+import "gitlab.com/elixxir/xxdk-wasm/worker"
+
+// List of tags that can be used when sending a message or registering a handler
+// to receive a message.
+const (
+	NewStateTag worker.Tag = "NewState"
+	SetTag      worker.Tag = "Set"
+	GetTag      worker.Tag = "Get"
+)