From 2b06d49188fc87f3bd7b61f9526816e701ef01d2 Mon Sep 17 00:00:00 2001 From: Jono Wenger <jono@elixxir.io> Date: Fri, 21 Jan 2022 12:18:27 -0800 Subject: [PATCH] Refactor restricted usernames and add file watcher to allow updating of list on the fly --- README.md | 16 +- banned/manager.go | 105 ------------ banned/manager_test.go | 100 ------------ cmd/params.go | 28 +--- cmd/root.go | 8 +- go.mod | 1 + interfaces/params/general.go | 12 +- io/manager.go | 12 +- io/manager_test.go | 10 +- io/userRegistration.go | 8 +- io/userRegistration_test.go | 51 +++--- restricted/manager.go | 299 +++++++++++++++++++++++++++++++++++ restricted/manager_test.go | 273 ++++++++++++++++++++++++++++++++ udb.yaml | 2 +- 14 files changed, 637 insertions(+), 288 deletions(-) delete mode 100644 banned/manager.go delete mode 100644 banned/manager_test.go create mode 100644 restricted/manager.go create mode 100644 restricted/manager_test.go diff --git a/README.md b/README.md index 4ab64c8..3f6079b 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,17 @@ twilioSid: "sid" twilioToken: "token" twilioVerification: "verification" -# Banned users which follow the regex codepath. -# Usernames should be separated by a Linux newline character ("\n"). -bannedRegexList: "bannedRegexList.txt" -# Simple banned username list. Any name exactly matching in this list will not be allowed as a username. -# Usernames should be separated by a Linux newline character ("\n"). -bannedUserList: "bannedUserList.txt" +# Path to line-seperated list of restricted usernames. Any username that appears +# on this list cannot be registered with UD. Each username on the list must be +# seperated by a new line character (\n). +restrictedUserList: "restrictedUserList.txt" + +# Path to line-seperated list of restricted username regular expressions. Any +# username that matches a statement on this list cannot be registered with UD. +# Each regular expressions on the list must be seperated by a new line character +# (\n). +restrictedRegexList: "restrictedRegexList.txt" ``` diff --git a/banned/manager.go b/banned/manager.go deleted file mode 100644 index 0f088f6..0000000 --- a/banned/manager.go +++ /dev/null @@ -1,105 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// Copyright © 2020 xx network SEZC // -// // -// Use of this source code is governed by a license that can be found in the // -// LICENSE file // -/////////////////////////////////////////////////////////////////////////////// - -package banned - -import ( - "github.com/pkg/errors" - jww "github.com/spf13/jwalterweatherman" - "gitlab.com/elixxir/user-discovery-bot/validation" - "regexp" - "strings" - "testing" -) - -// Manager contains two lists of banned/reserved usernames. It handles -// checking for banned usernames on user registration. -type Manager struct { - // A simple banned user lookup. Any user that exactly matches something - // in this map will be considered banned/reserved. - bannedUserList map[string]struct{} - - // A more complex banned user lookup. Contains a list of regular expressions, - // and any username which matches any of these regular expressions will be - // considered banned/reserved. - bannedRegexList []*regexp.Regexp -} - -// NewManager constructs the banned.Manager object. NewManager is passed in -// two text files containing a list where values are separated by the -// Linux newline ("\n"). NewManager will parse these two lists separately to create a -// Manager.bannedUserList and a Manager.bannedRegexList respectively. -func NewManager(bannedUserFile, bannedRegexFile string) (*Manager, error) { - // Construct a map of banned/reserved usernames - bannedUsers := make(map[string]struct{}) - if bannedUserFile != "" { - bannedUserList := strings.Split(bannedUserFile, "\n") - for _, bannedUser := range bannedUserList { - if bannedUser == "" { // Skip any empty lines - continue - } - bannedUsers[validation.Canonicalize(bannedUser)] = struct{}{} - } - } - - // Construct a regex list for banned/reserved usernames - bannedRegexList := make([]*regexp.Regexp, 0) - if bannedRegexFile != "" { - regexList := strings.Split(bannedRegexFile, "\n") - for _, bannedRegex := range regexList { - if bannedRegex == "" { // Skip any empty lines - continue - } - - // Compile regex expression - regex, err := regexp.Compile(bannedRegex) - if err != nil { - return nil, errors.Errorf("Failed to compile banned user regex %q: %v", bannedRegex, err) - } - - bannedRegexList = append(bannedRegexList, regex) - - } - } - - return &Manager{ - bannedUserList: bannedUsers, - bannedRegexList: bannedRegexList, - }, nil -} - -// IsBanned checks if the username is in Manager's bannedUserList or -// matched to any banned regular expression. -func (m *Manager) IsBanned(username string) bool { - _, exists := m.bannedUserList[username] - if exists { - return exists - } - - return m.isRegexBanned(username) -} - -// isRegexBanned checks is the username matches any banned regular expression. -func (m *Manager) isRegexBanned(username string) bool { - for _, regex := range m.bannedRegexList { - if regex.MatchString(username) { - return true - } - } - - return false -} - -// SetBannedTest is a testing only helper function which sets a username -// in Manager's bannedUserList. -func (m *Manager) SetBannedTest(username string, t *testing.T) { - if t == nil { - jww.FATAL.Panic("Cannot use this outside of testing") - } - - m.bannedUserList[username] = struct{}{} -} diff --git a/banned/manager_test.go b/banned/manager_test.go deleted file mode 100644 index 128284e..0000000 --- a/banned/manager_test.go +++ /dev/null @@ -1,100 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// Copyright © 2020 xx network SEZC // -// // -// Use of this source code is governed by a license that can be found in the // -// LICENSE file // -/////////////////////////////////////////////////////////////////////////////// - -package banned - -import ( - "gitlab.com/elixxir/user-discovery-bot/validation" - "reflect" - "regexp" - "testing" -) - -func TestNewManager(t *testing.T) { - expectedManager := &Manager{ - bannedUserList: map[string]struct{}{ - "privategrity": {}, - "privategrity_corp": {}, - }, - bannedRegexList: []*regexp.Regexp{ - regexp.MustCompile("xx"), - regexp.MustCompile("xx.*?network"), - }, - } - - bannedUserList := "" - for key := range expectedManager.bannedUserList { - bannedUserList += key + "\n" - } - - bannedRegexList := "" - for _, regex := range expectedManager.bannedRegexList { - bannedRegexList += regex.String() + "\n" - } - - m, err := NewManager(bannedUserList, bannedRegexList) - if err != nil { - t.Fatalf("NewManager error: %v", err) - } - - if !reflect.DeepEqual(m, expectedManager) { - t.Errorf("Constructed manager does not match expected output."+ - "\nExpected: %+v"+ - "\nReceived: %+v", expectedManager, m) - } -} - -func TestManager_IsBanned_GoodUsername(t *testing.T) { - bannedUserList := "Privategrity\nPrivategrity_Corp" - bannedRegexList := "xx\nxx.*?network" - - m, err := NewManager(bannedUserList, bannedRegexList) - if err != nil { - t.Fatalf("NewManager error: %v", err) - } - - goodUsernames := []string{ - "john_doe", - "private", - "network", - "Privategrity!??!Corporation", - } - - for _, goodUsername := range goodUsernames { - if m.IsBanned(goodUsername) { - t.Errorf("Username %q was recognized as banned when it should not be", goodUsername) - } - - } - -} - -// -func TestManager_IsBanned_BadUsername(t *testing.T) { - bannedUserList := "Privategrity\nPrivategrity_Corp" - bannedRegexList := "xx\nxx.*?network" - - m, err := NewManager(bannedUserList, bannedRegexList) - if err != nil { - t.Fatalf("NewManager error: %v", err) - } - - badUsernames := []string{ - "xxfsdfsdfsdklfjnetwork", - "Privategrity", - "Privategrity_Corp", - "exxplostion", - } - - for _, badUsername := range badUsernames { - if !m.IsBanned(validation.Canonicalize(badUsername)) { - t.Errorf("Username %q was not recognized as banned when it should be", badUsername) - } - - } - -} diff --git a/cmd/params.go b/cmd/params.go index 88d166a..bb9ae57 100644 --- a/cmd/params.go +++ b/cmd/params.go @@ -45,18 +45,6 @@ func InitParams(vip *viper.Viper) params.General { jww.FATAL.Fatalf("Failed to read session path: %+v", err) } - // Load banned user list - bannedUserList, err := utils.ReadFile(viper.GetString("bannedUserList")) - if err != nil { - jww.WARN.Printf("Failed to read banned user list: %v", err) - } - - // Load banned regex list - bannedRegexList, err := utils.ReadFile(viper.GetString("bannedRegexList")) - if err != nil { - jww.WARN.Printf("Failed to read banned regex list: %v", err) - } - // Only require proto user path if session does not exist var protoUserJson []byte protoUserPath := "" @@ -111,13 +99,13 @@ func InitParams(vip *viper.Viper) params.General { jww.INFO.Printf("UDB port: %s", ioparams.Port) return params.General{ - PermCert: permCert, - SessionPath: sessionPath, - Database: dbparams, - IO: ioparams, - Twilio: twilioparams, - ProtoUserJson: protoUserJson, - BannedUserList: string(bannedUserList), - BannedRegexList: string(bannedRegexList), + PermCert: permCert, + SessionPath: sessionPath, + Database: dbparams, + IO: ioparams, + Twilio: twilioparams, + ProtoUserJson: protoUserJson, + RestrictedUserListPath: viper.GetString("restrictedUserList"), + RestrictedRegexListPath: viper.GetString("restrictedRegexList"), } } diff --git a/cmd/root.go b/cmd/root.go index 2f2797b..feec4c2 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -9,9 +9,9 @@ import ( "gitlab.com/elixxir/client/interfaces/params" "gitlab.com/elixxir/client/single" "gitlab.com/elixxir/comms/mixmessages" - "gitlab.com/elixxir/user-discovery-bot/banned" "gitlab.com/elixxir/user-discovery-bot/cmix" "gitlab.com/elixxir/user-discovery-bot/io" + "gitlab.com/elixxir/user-discovery-bot/restricted" "gitlab.com/elixxir/user-discovery-bot/storage" "gitlab.com/elixxir/user-discovery-bot/twilio" "gitlab.com/xx_network/comms/connect" @@ -67,13 +67,15 @@ var rootCmd = &cobra.Command{ } permCert, err := tls.ExtractPublicKey(cert) - bannedManager, err := banned.NewManager(p.BannedUserList, p.BannedRegexList) + restrictedManager, err := restricted.NewManager( + p.RestrictedUserListPath, p.RestrictedRegexListPath, nil) if err != nil { jww.FATAL.Panicf("Failed to construct ban manager: %v", err) } // Set up manager with the ability to contact permissioning - manager := io.NewManager(p.IO, &id.UDB, permCert, twilioManager, bannedManager, storage) + manager := io.NewManager(p.IO, &id.UDB, permCert, twilioManager, + restrictedManager, storage) hostParams := connect.GetDefaultHostParams() hostParams.AuthEnabled = false permHost, err := manager.Comms.AddHost(&id.Permissioning, diff --git a/go.mod b/go.mod index b79c193..6d931bc 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module gitlab.com/elixxir/user-discovery-bot go 1.13 require ( + github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/golang/protobuf v1.5.2 github.com/magiconair/properties v1.8.5 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect diff --git a/interfaces/params/general.go b/interfaces/params/general.go index dad5898..1fdc52a 100644 --- a/interfaces/params/general.go +++ b/interfaces/params/general.go @@ -9,12 +9,12 @@ package params type General struct { - SessionPath string - ProtoUserJson []byte - Ndf string - PermCert []byte - BannedUserList string - BannedRegexList string + SessionPath string + ProtoUserJson []byte + Ndf string + PermCert []byte + RestrictedUserListPath string // Path to list of line-seperated usernames + RestrictedRegexListPath string // Path to list of line-seperated regexes Database IO diff --git a/io/manager.go b/io/manager.go index e4ea3d8..1c032c4 100644 --- a/io/manager.go +++ b/io/manager.go @@ -13,8 +13,8 @@ import ( "fmt" pb "gitlab.com/elixxir/comms/mixmessages" "gitlab.com/elixxir/comms/udb" - "gitlab.com/elixxir/user-discovery-bot/banned" "gitlab.com/elixxir/user-discovery-bot/interfaces/params" + "gitlab.com/elixxir/user-discovery-bot/restricted" "gitlab.com/elixxir/user-discovery-bot/storage" "gitlab.com/elixxir/user-discovery-bot/twilio" "gitlab.com/xx_network/comms/messages" @@ -28,17 +28,18 @@ type Manager struct { PermissioningPublicKey *rsa.PublicKey Storage *storage.Storage Twilio *twilio.Manager - Banned *banned.Manager + Restricted *restricted.Manager } // Create a new UserDiscovery Manager given a set of Params func NewManager(p params.IO, id *id.ID, permissioningCert *rsa.PublicKey, - twilio *twilio.Manager, banned *banned.Manager, storage *storage.Storage) *Manager { + twilio *twilio.Manager, restricted *restricted.Manager, + storage *storage.Storage) *Manager { m := &Manager{ Storage: storage, PermissioningPublicKey: permissioningCert, Twilio: twilio, - Banned: banned, + Restricted: restricted, } m.Comms = udb.StartServer(id, fmt.Sprintf("0.0.0.0:%s", p.Port), newImplementation(m), p.Cert, p.Key) @@ -50,7 +51,8 @@ func newImplementation(m *Manager) *udb.Implementation { impl := udb.NewImplementation() impl.Functions.RegisterUser = func(registration *pb.UDBUserRegistration) (*messages.Ack, error) { - return registerUser(registration, m.PermissioningPublicKey, m.Storage, m.Banned) + return registerUser( + registration, m.PermissioningPublicKey, m.Storage, m.Restricted) } impl.Functions.RemoveUser = func(msg *pb.FactRemovalRequest) (*messages.Ack, error) { diff --git a/io/manager_test.go b/io/manager_test.go index dac873d..d9cc794 100644 --- a/io/manager_test.go +++ b/io/manager_test.go @@ -1,8 +1,8 @@ package io import ( - "gitlab.com/elixxir/user-discovery-bot/banned" "gitlab.com/elixxir/user-discovery-bot/interfaces/params" + "gitlab.com/elixxir/user-discovery-bot/restricted" "gitlab.com/elixxir/user-discovery-bot/storage" "gitlab.com/elixxir/user-discovery-bot/twilio" "gitlab.com/xx_network/primitives/id" @@ -18,12 +18,10 @@ func TestNewManager(t *testing.T) { } store := storage.NewTestDB(t) tm := twilio.NewMockManager(store) - bannedManager, err := banned.NewManager("", "") - if err != nil { - t.Fatalf("Failed to construct ban manager: %v", err) - } + restrictedUsernames := restricted.NewManagerForTesting(nil, nil, t) - m := NewManager(p, id.NewIdFromString("zezima", id.User, t), nil, tm, bannedManager, store) + m := NewManager(p, id.NewIdFromString("zezima", id.User, t), nil, tm, + restrictedUsernames, store) if m == nil || reflect.TypeOf(m) != reflect.TypeOf(&Manager{}) { t.Errorf("Did not receive a manager") } diff --git a/io/userRegistration.go b/io/userRegistration.go index 38fe6c3..69a5e54 100644 --- a/io/userRegistration.go +++ b/io/userRegistration.go @@ -14,7 +14,7 @@ import ( "gitlab.com/elixxir/crypto/hash" "gitlab.com/elixxir/crypto/registration" "gitlab.com/elixxir/primitives/fact" - "gitlab.com/elixxir/user-discovery-bot/banned" + "gitlab.com/elixxir/user-discovery-bot/restricted" "gitlab.com/elixxir/user-discovery-bot/storage" "gitlab.com/elixxir/user-discovery-bot/validation" "gitlab.com/xx_network/comms/messages" @@ -25,7 +25,7 @@ import ( // Endpoint which handles a users attempt to register func registerUser(msg *pb.UDBUserRegistration, permPublicKey *rsa.PublicKey, - store *storage.Storage, bannedManager *banned.Manager) (*messages.Ack, error) { + store *storage.Storage, restrictedManager *restricted.Manager) (*messages.Ack, error) { // Nil checks if msg == nil || msg.Frs == nil || msg.Frs.Fact == nil || @@ -49,8 +49,8 @@ func registerUser(msg *pb.UDBUserRegistration, permPublicKey *rsa.PublicKey, return nil, errors.Errorf("Username %q is invalid: %v", username, err) } - // Check if the username is banned - if bannedManager.IsBanned(canonicalUsername) { + // Check if the username is restricted + if restrictedManager.IsRestricted(canonicalUsername) { // Return same error message as if the user was already taken return &messages.Ack{}, errors.Errorf("Username %s is already taken. "+ "Please try again", username) diff --git a/io/userRegistration_test.go b/io/userRegistration_test.go index 0b11e55..53d92c8 100644 --- a/io/userRegistration_test.go +++ b/io/userRegistration_test.go @@ -15,7 +15,7 @@ import ( "gitlab.com/elixxir/crypto/hash" "gitlab.com/elixxir/crypto/registration" "gitlab.com/elixxir/primitives/fact" - "gitlab.com/elixxir/user-discovery-bot/banned" + "gitlab.com/elixxir/user-discovery-bot/restricted" "gitlab.com/elixxir/user-discovery-bot/storage" "gitlab.com/elixxir/user-discovery-bot/validation" "gitlab.com/xx_network/crypto/signature/rsa" @@ -113,12 +113,9 @@ func TestRegisterUser(t *testing.T) { t.FailNow() } - bannedManager, err := banned.NewManager("", "") - if err != nil { - t.Fatalf("Failed to construct ban manager: %v", err) - } + restrictedUsernames := restricted.NewManagerForTesting(nil, nil, t) - _, err = registerUser(registerMsg, cert, store, bannedManager) + _, err = registerUser(registerMsg, cert, store, restrictedUsernames) if err != nil { t.Errorf("Failed happy path: %v", err) } @@ -168,9 +165,9 @@ func TestRegisterUser(t *testing.T) { } -// TestRegisterUser_Banned tests that registering a username in the banned list -// returns an error. -func TestRegisterUser_Banned(t *testing.T) { +// TestRegisterUser_Restricted tests that registering a username in the restricted +// list returns an error. +func TestRegisterUser_Restricted(t *testing.T) { // Initialize client and storage clientId, clientKey := initClientFields(t) store := storage.NewTestDB(t) @@ -193,12 +190,8 @@ func TestRegisterUser_Banned(t *testing.T) { t.FailNow() } - bannedManager, err := banned.NewManager(validation.Canonicalize(registerMsg.IdentityRegistration.Username), "") - if err != nil { - t.Fatalf("Failed to construct ban manager: %v", err) - } - - _, err = registerUser(registerMsg, cert, store, bannedManager) + restrictedUsernames := restricted.NewManagerForTesting(map[string]struct{}{validation.Canonicalize(registerMsg.IdentityRegistration.Username): {}}, nil, t) + _, err = registerUser(registerMsg, cert, store, restrictedUsernames) if err == nil { t.Errorf("Failed happy path: %v", err) } @@ -223,11 +216,8 @@ func TestRegisterUser_InvalidSignatures(t *testing.T) { t.Fatalf("Could not parse precanned time: %v", err.Error()) } - // Construct dummy ban manager - bannedManager, err := banned.NewManager("", "") - if err != nil { - t.Fatalf("Failed to construct ban manager: %v", err) - } + // Construct dummy restricted username manager + restrictedUsernames := restricted.NewManagerForTesting(nil, nil, t) // Set an invalid identity signature, check that error occurred registerMsg, err := buildUserRegistrationMessage(clientId, clientKey, testTime, t) @@ -235,7 +225,7 @@ func TestRegisterUser_InvalidSignatures(t *testing.T) { t.FailNow() } registerMsg.IdentitySignature = []byte("invalid") - _, err = registerUser(registerMsg, cert, store, bannedManager) + _, err = registerUser(registerMsg, cert, store, restrictedUsernames) if err == nil { t.Errorf("Should not be able to verify identity signature: %v", err) } @@ -246,7 +236,7 @@ func TestRegisterUser_InvalidSignatures(t *testing.T) { t.FailNow() } registerMsg.Frs.FactSig = []byte("invalid") - _, err = registerUser(registerMsg, cert, store, bannedManager) + _, err = registerUser(registerMsg, cert, store, restrictedUsernames) if err == nil { t.Errorf("Should not be able to verify fact signature: %v", err) } @@ -257,7 +247,7 @@ func TestRegisterUser_InvalidSignatures(t *testing.T) { t.FailNow() } registerMsg.PermissioningSignature = []byte("invalid") - _, err = registerUser(registerMsg, cert, store, bannedManager) + _, err = registerUser(registerMsg, cert, store, restrictedUsernames) if err == nil { t.Errorf("Should not be able to verify permissioning signature: %v", err) } @@ -282,11 +272,8 @@ func TestRegisterUser_InvalidMessage(t *testing.T) { t.Fatalf("Could not parse precanned time: %v", err.Error()) } - // Construct dummy ban manager - bannedManager, err := banned.NewManager("", "") - if err != nil { - t.Fatalf("Failed to construct ban manager: %v", err) - } + // Construct restricted username manager + restrictedManager := restricted.NewManagerForTesting(nil, nil, t) // Set an invalid message, check that error occurred registerMsg, err := buildUserRegistrationMessage(clientId, clientKey, testTime, t) @@ -294,7 +281,7 @@ func TestRegisterUser_InvalidMessage(t *testing.T) { t.FailNow() } registerMsg = nil - _, err = registerUser(registerMsg, cert, store, bannedManager) + _, err = registerUser(registerMsg, cert, store, restrictedManager) if err == nil { t.Errorf("Should not be able to handle nil message: %v", err) } @@ -305,7 +292,7 @@ func TestRegisterUser_InvalidMessage(t *testing.T) { t.FailNow() } registerMsg.Frs = nil - _, err = registerUser(registerMsg, cert, store, bannedManager) + _, err = registerUser(registerMsg, cert, store, restrictedManager) if err == nil { t.Errorf("Should not be able to handle nil FactRegistration message: %v", err) } @@ -316,7 +303,7 @@ func TestRegisterUser_InvalidMessage(t *testing.T) { t.FailNow() } registerMsg.Frs.Fact = nil - _, err = registerUser(registerMsg, cert, store, bannedManager) + _, err = registerUser(registerMsg, cert, store, restrictedManager) if err == nil { t.Errorf("Should not be able to handle nil Fact message: %v", err) } @@ -327,7 +314,7 @@ func TestRegisterUser_InvalidMessage(t *testing.T) { t.FailNow() } registerMsg.IdentityRegistration = nil - _, err = registerUser(registerMsg, cert, store, bannedManager) + _, err = registerUser(registerMsg, cert, store, restrictedManager) if err == nil { t.Errorf("Should not be able to handle nil IdentityRegistration message: %v", err) } diff --git a/restricted/manager.go b/restricted/manager.go new file mode 100644 index 0000000..bf18205 --- /dev/null +++ b/restricted/manager.go @@ -0,0 +1,299 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright © 2020 xx network SEZC // +// // +// Use of this source code is governed by a license that can be found in the // +// LICENSE file // +/////////////////////////////////////////////////////////////////////////////// + +package restricted + +import ( + "bufio" + "github.com/fsnotify/fsnotify" + "github.com/pkg/errors" + jww "github.com/spf13/jwalterweatherman" + "gitlab.com/elixxir/user-discovery-bot/validation" + "os" + "regexp" + "strings" + "sync" + "testing" +) + +// Manager contains the list of restricted usernames and regex in memory and +// handles the checking if registered usernames are restricted. Also handles +// the dynamic updating of the lists in memory when their source file changes. +type Manager struct { + // File path to the restricted username list + usernamePath string + + // File path to the restricted regex list + regexPath string + + // List of restricted usernames that are not allowed to be registered + usernames map[string]struct{} + + // List of regular expressions that registered usernames cannot match + regexes []*regexp.Regexp + + mux sync.RWMutex +} + +// NewManager initialises the restricted Manager with the contents of the +// username and regex lists. Also starts a thread that dynamically updates the +// lists on file change. +func NewManager(usernamePath, regexPath string, quit chan struct{}) (*Manager, error) { + // Construct a map of restricted usernames + usernames, err := usernameListParser(usernamePath) + if err != nil { + return nil, err + } + + // Construct list of restricted regex + regexes, err := regexListParser(regexPath) + if err != nil { + return nil, err + } + + // Construct the restricted username manager + m := &Manager{ + usernamePath: usernamePath, + regexPath: regexPath, + usernames: usernames, + regexes: regexes, + } + + // Start the file watcher + err = m.fileWatch(quit) + if err != nil { + return nil, err + } + + return m, nil +} + +// usernameListParser reads the list of restricted usernames from the given +// filepath and returns a map of them. Usernames must be seperated by new lines. +// Usernames are canonicalized before added to the map. +func usernameListParser(path string) (map[string]struct{}, error) { + // Open the file + f, err := os.Open(path) + if err != nil { + return nil, err + } + + // Scan the file line by line and add each username to the list + scanner := bufio.NewScanner(f) + usernames := make(map[string]struct{}) + for scanner.Scan() { + if line := strings.TrimSpace(scanner.Text()); line != "" { + usernames[validation.Canonicalize(line)] = struct{}{} + } + } + + if err = scanner.Err(); err != nil { + _ = f.Close() // Ignore error; scanner error takes precedence + return nil, err + } + + err = f.Close() + if err != nil { + return nil, err + } + + return usernames, nil +} + +// regexListParser reads the list of restricted regular expressions from the +// given filepath and returns a list of them compiled. Regular expressions must +// be seperated by new lines. +func regexListParser(path string) ([]*regexp.Regexp, error) { + // Open the file + f, err := os.Open(path) + if err != nil { + return nil, err + } + + // Scan the file line by line and add each regex to the list + scanner := bufio.NewScanner(f) + var regexes []*regexp.Regexp + for scanner.Scan() { + if line := strings.TrimSpace(scanner.Text()); line != "" { + regex, err := regexp.Compile(line) + if err != nil { + _ = f.Close() // Ignore error; regex error takes precedence + return nil, err + } + regexes = append(regexes, regex) + } + } + + if err = scanner.Err(); err != nil { + _ = f.Close() // Ignore error; scanner error takes precedence + return nil, err + } + + err = f.Close() + if err != nil { + return nil, err + } + + return regexes, nil +} + +func (m *Manager) updateLists() error { + // Get new username list + usernames, err := usernameListParser(m.usernamePath) + if err != nil { + return err + } + + // Get new regex list + regexes, err := regexListParser(m.regexPath) + if err != nil { + return err + } + + m.mux.Lock() + defer m.mux.Unlock() + + m.usernames = usernames + m.regexes = regexes + + return nil +} + +func (m *Manager) updateUsernamesFromFile() error { + // Get new username list + usernames, err := usernameListParser(m.usernamePath) + if err != nil { + return err + } + + m.mux.Lock() + defer m.mux.Unlock() + + m.usernames = usernames + + return nil +} + +func (m *Manager) updateRegexesFromFile() error { + // Get new regex list + regexes, err := regexListParser(m.regexPath) + if err != nil { + return err + } + + m.mux.Lock() + defer m.mux.Unlock() + + m.regexes = regexes + + return nil +} + +// IsRestricted checks if the username is matches any restricted usernames or +// restricted regular expressions. Usernames must be canonicalized before they +// are passed in. +func (m *Manager) IsRestricted(username string) bool { + m.mux.RLock() + defer m.mux.RUnlock() + + _, exists := m.usernames[username] + if exists { + return exists + } + + return m.matchRestrictedRegex(username) +} + +// matchRestrictedRegex checks if the username matches any of the restricted +// regular expressions. +func (m *Manager) matchRestrictedRegex(username string) bool { + for _, regex := range m.regexes { + if regex.MatchString(username) { + return true + } + } + + return false +} + +// fileWatch watches for changes to restricted username list files and +// dynamically updates the list in memory when the files change. +func (m *Manager) fileWatch(quit chan struct{}) error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return errors.Errorf("failed to initialize watcher for restricted username lists: %+v", err) + } + + go func() { + jww.INFO.Print("Starting restricted username file watcher.") + defer func() { + err = watcher.Close() + if err != nil { + jww.ERROR.Printf("Failed to close restricted username file watcher: %+v", err) + } + }() + + for { + select { + case <-quit: + jww.INFO.Print("Quitting restricted username file watcher.") + return + case event := <-watcher.Events: + jww.DEBUG.Printf("Restricted username file watcher: file %q op %s", event.Name, event.Op) + if event.Op == fsnotify.Write || event.Op == fsnotify.Create { + if strings.Contains(m.usernamePath, event.Name) { + jww.INFO.Printf("Updating restricted usernames from file %q.", event.Name) + err = m.updateUsernamesFromFile() + if err != nil { + jww.ERROR.Printf("Failed to update restricted username list: %+v", err) + } + } else if strings.Contains(m.regexPath, event.Name) { + jww.INFO.Printf("Updating restricted regex from file %q.", event.Name) + err = m.updateRegexesFromFile() + if err != nil { + jww.ERROR.Printf("Failed to update restricted regex list: %+v", err) + } + } + } else if event.Op == fsnotify.Remove { + jww.ERROR.Printf("Restricted username file watcher: %q was deleted", event.Name) + } + case err := <-watcher.Errors: + jww.ERROR.Printf("Restricted username file watcher encountered an error: %+v", err) + } + } + }() + + err = watcher.Add(m.usernamePath) + if err != nil { + return errors.Errorf("could not add %q to restricted username file watcher: %+v", m.usernamePath, err) + } + + err = watcher.Add(m.regexPath) + if err != nil { + return errors.Errorf("could not add %q to restricted username file watcher: %+v", m.regexPath, err) + } + return nil +} + +// NewManagerForTesting creates a new Manager without a file backend to only be +// used for testing. +func NewManagerForTesting(usernames map[string]struct{}, + regexes []*regexp.Regexp, x interface{}) *Manager { + switch x.(type) { + case *testing.T, *testing.M, *testing.B, *testing.PB: + break + default: + jww.FATAL.Panicf("NewManagerForTesting can only be used for testing.") + } + + return &Manager{ + usernamePath: "", + regexPath: "", + usernames: usernames, + regexes: regexes, + } +} diff --git a/restricted/manager_test.go b/restricted/manager_test.go new file mode 100644 index 0000000..72146bc --- /dev/null +++ b/restricted/manager_test.go @@ -0,0 +1,273 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright © 2020 xx network SEZC // +// // +// Use of this source code is governed by a license that can be found in the // +// LICENSE file // +/////////////////////////////////////////////////////////////////////////////// + +package restricted + +import ( + "gitlab.com/elixxir/user-discovery-bot/validation" + "gitlab.com/xx_network/primitives/utils" + "os" + "reflect" + "regexp" + "strconv" + "testing" + "time" +) + +// Tests that NewManager returns a new Manager with the expected values. +func TestNewManager(t *testing.T) { + usernamesPath := "restrictedUsernames.txt" + regexPath := "restrictedRegex.txt" + expectedManager := &Manager{ + usernamePath: usernamesPath, + regexPath: regexPath, + usernames: map[string]struct{}{ + "privategrity": {}, + "privategrity_corp": {}, + }, + regexes: []*regexp.Regexp{ + regexp.MustCompile("xx"), + regexp.MustCompile("xx.*?network"), + }, + } + + usernameList := usernamesToList(expectedManager.usernames) + regexList := regexesToList(expectedManager.regexes) + + err := utils.WriteFile( + usernamesPath, []byte(usernameList), utils.FilePerms, utils.DirPerms) + if err != nil { + t.Errorf("Failed to write file: %+v", err) + } + err = utils.WriteFile( + regexPath, []byte(regexList), utils.FilePerms, utils.DirPerms) + if err != nil { + t.Errorf("Failed to write file: %+v", err) + } + defer func() { + err = os.RemoveAll(usernamesPath) + if err != nil { + t.Errorf("Error deleting test file %q: %+v", usernamesPath, err) + } + err = os.RemoveAll(regexPath) + if err != nil { + t.Errorf("Error deleting test file %q: %+v", regexPath, err) + } + }() + + quit := make(chan struct{}) + m, err := NewManager(usernamesPath, regexPath, quit) + if err != nil { + t.Errorf("NewManager returned an error: %+v", err) + } + + if !reflect.DeepEqual(m, expectedManager) { + t.Errorf("New manager does not match expected."+ + "\nexpected: %+v\nreceived: %+v", expectedManager, m) + } + + quit <- struct{}{} +} + +// Tests that Manager.IsRestricted returns false for a list of known non- +// restricted usernames. +func TestManager_IsRestricted_GoodUsernames(t *testing.T) { + usernameList := "Privategrity\nPrivategrity_Corp" + regexList := "xx\nxx.*?network" + + m, deleteFunc := newTestManager(usernameList, regexList, t) + defer deleteFunc() + + usernames := []string{ + "john_doe", + "private", + "network", + "Privategrity!??!Corporation", + } + + for _, username := range usernames { + if m.IsRestricted(username) { + t.Errorf("Username %q was recognized as restricted when it "+ + "should not be.", username) + } + } +} + +// Tests that Manager.IsRestricted returns true for a list of known restricted +// usernames. +func TestManager_IsRestricted_BadUsernames(t *testing.T) { + usernameList := "Privategrity\nPrivategrity_Corp" + regexList := "xx\nxx.*?network" + + m, deleteFunc := newTestManager(usernameList, regexList, t) + defer deleteFunc() + + usernames := []string{ + "xxfsdfsdfsdklfjnetwork", + "Privategrity", + "Privategrity_Corp", + "exxplostion", + } + + for _, username := range usernames { + if !m.IsRestricted(validation.Canonicalize(username)) { + t.Errorf("Username %q was not recognized as restricted when it "+ + "should have been.", username) + } + } +} + +func TestManager_fileWatch(t *testing.T) { + usernames := map[string]struct{}{ + "privategrity": {}, + "privategrity_corp": {}, + } + + regexes := []*regexp.Regexp{ + regexp.MustCompile("xx"), + regexp.MustCompile("xx.*?network"), + } + + // Create manager and check it initialised the files correctly. + m, deleteFunc := newTestManager( + usernamesToList(usernames), regexesToList(regexes), t) + defer deleteFunc() + + if !reflect.DeepEqual(m.usernames, usernames) { + t.Errorf("Usernames in memory do not match usernames in file."+ + "\nexpected: %v\nreceived: %v", usernames, m.usernames) + } + if !reflect.DeepEqual(m.regexes, regexes) { + t.Errorf("Regexes in memory do not match regexes in file."+ + "\nexpected: %v\nreceived: %v", regexes, m.regexes) + } + + // Modify each file + usernames = map[string]struct{}{ + "privategrity": {}, + "privategrity_corp": {}, + "xxnetwork": {}, + } + + regexes = []*regexp.Regexp{ + regexp.MustCompile("xx.*?network"), + regexp.MustCompile("david.*?chaum"), + } + + err := utils.WriteFile(m.usernamePath, []byte(usernamesToList(usernames)), + utils.FilePerms, utils.DirPerms) + if err != nil { + t.Errorf("Failed to write username file: %+v", err) + } + err = utils.WriteFile(m.regexPath, []byte(regexesToList(regexes)), + utils.FilePerms, utils.DirPerms) + if err != nil { + t.Errorf("Failed to write regex file: %+v", err) + } + + time.Sleep(100 * time.Millisecond) + + // Check that the lists in memory match the new files + if !reflect.DeepEqual(m.usernames, usernames) { + t.Errorf("Usernames in memory do not match usernames in file."+ + "\nexpected: %v\nreceived: %v", usernames, m.usernames) + } + if !reflect.DeepEqual(m.regexes, regexes) { + t.Errorf("Regexes in memory do not match regexes in file."+ + "\nexpected: %v\nreceived: %v", regexes, m.regexes) + } + + // Modify each file + usernames = map[string]struct{}{} + + regexes = []*regexp.Regexp{ + regexp.MustCompile("hi"), + } + + err = utils.WriteFile(m.usernamePath, []byte(usernamesToList(usernames)), + utils.FilePerms, utils.DirPerms) + if err != nil { + t.Errorf("Failed to write username file: %+v", err) + } + err = utils.WriteFile(m.regexPath, []byte(regexesToList(regexes)), + utils.FilePerms, utils.DirPerms) + if err != nil { + t.Errorf("Failed to write regex file: %+v", err) + } + + time.Sleep(100 * time.Millisecond) + + // Check that the lists in memory match the new files + if !reflect.DeepEqual(m.usernames, usernames) { + t.Errorf("Usernames in memory do not match usernames in file."+ + "\nexpected: %v\nreceived: %v", usernames, m.usernames) + } + if !reflect.DeepEqual(m.regexes, regexes) { + t.Errorf("Regexes in memory do not match regexes in file."+ + "\nexpected: %v\nreceived: %v", regexes, m.regexes) + } +} + +// newTestManager creates two files for each list and loads them into a new +// manager. A function is returned to remove the files after the test. +func newTestManager(usernameList, regexList string, t *testing.T) ( + *Manager, func()) { + timeNow := strconv.Itoa(int(time.Now().UnixNano())) + usernamesPath := "restrictedUsernames-" + timeNow + ".txt" + regexPath := "restrictedRegex-" + timeNow + ".txt" + err := utils.WriteFile( + usernamesPath, []byte(usernameList), utils.FilePerms, utils.DirPerms) + if err != nil { + t.Errorf("Failed to write file: %+v", err) + } + + err = utils.WriteFile( + regexPath, []byte(regexList), utils.FilePerms, utils.DirPerms) + if err != nil { + t.Errorf("Failed to write file: %+v", err) + } + + quit := make(chan struct{}) + deleteFunc := func() { + quit <- struct{}{} + err = os.RemoveAll(usernamesPath) + if err != nil { + t.Errorf("Error deleting test file %q: %+v", usernamesPath, err) + } + err = os.RemoveAll(regexPath) + if err != nil { + t.Errorf("Error deleting test file %q: %+v", regexPath, err) + } + } + + m, err := NewManager(usernamesPath, regexPath, quit) + if err != nil { + t.Errorf("NewManager returned an error: %+v", err) + } + + return m, deleteFunc +} + +// usernamesToList converts the map of usernames to line-seperated list string. +func usernamesToList(usernames map[string]struct{}) string { + usernameList := "" + for username := range usernames { + usernameList += username + "\n" + } + + return usernameList +} + +// regexesToList converts a list of regexp.Regexp to line-seperated list string. +func regexesToList(regexes []*regexp.Regexp) string { + regexList := "" + for _, regex := range regexes { + regexList += regex.String() + "\n" + } + + return regexList +} diff --git a/udb.yaml b/udb.yaml index 2fb0265..4f5833e 100644 --- a/udb.yaml +++ b/udb.yaml @@ -11,4 +11,4 @@ dbPassword: "" dbName: "" dbAddress: "" -bannedUserList: "bannedUserList.csv" \ No newline at end of file +restrictedUserList: "restrictedUserList.csv" \ No newline at end of file -- GitLab