* further splitting onionrutils into a module

onionr/communicator.py

@@ -27,6 +27,7 @@ from communicatorutils import downloadblocks, lookupblocks, lookupadders
 from communicatorutils import servicecreator, connectnewpeers, uploadblocks
 from communicatorutils import daemonqueuehandler, announcenode, deniableinserts
 from communicatorutils import cooldownpeer, housekeeping, netcheck
+from onionrutils import localcommand
 from etc import humanreadabletime
 import onionrservices, onionr, onionrproofs
 
@@ -184,7 +185,7 @@ class OnionrCommunicatorDaemon:
         else:
             for server in self.service_greenlets:
                 server.stop()
-        self._core._utils.localCommand('shutdown') # shutdown the api
+        localcommand.local_command(self._core, 'shutdown') # shutdown the api
         time.sleep(0.5)
 
     def lookupAdders(self):
@@ -364,9 +365,9 @@ class OnionrCommunicatorDaemon:
 
     def detectAPICrash(self):
         '''exit if the api server crashes/stops'''
-        if self._core._utils.localCommand('ping', silent=False) not in ('pong', 'pong!'):
+        if localcommand.local_command(self._core, 'ping', silent=False) not in ('pong', 'pong!'):
             for i in range(300):
-                if self._core._utils.localCommand('ping') in ('pong', 'pong!') or self.shutdown:
+                if localcommand.local_command(self._core, 'ping') in ('pong', 'pong!') or self.shutdown:
                     break # break for loop
                 time.sleep(1)
             else:

onionr/communicatorutils/connectnewpeers.py

@@ -20,6 +20,7 @@
 import time, sys
 import onionrexceptions, logger, onionrpeers
 from utils import networkmerger
+from onionrutils import stringvalidators
 # secrets module was added into standard lib in 3.6+
 if sys.version_info[0] == 3 and sys.version_info[1] < 6:
     from dependencies import secrets
@@ -30,7 +31,7 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
     retData = False
     tried = comm_inst.offlinePeers
     if peer != '':
-        if comm_inst._core._utils.validateID(peer):
+        if stringvalidators.validate_transport(peer):
             peerList = [peer]
         else:
             raise onionrexceptions.InvalidAddress('Will not attempt connection test to invalid address')

onionr/communicatorutils/daemonqueuehandler.py

@@ -19,6 +19,7 @@
 '''
 import logger
 import onionrevents as events
+from onionrutils import localcommand
 def handle_daemon_commands(comm_inst):
     cmd = comm_inst._core.daemonQueue()
     response = ''
@@ -39,7 +40,7 @@ def handle_daemon_commands(comm_inst):
             if response == '':
                 response = 'none'
         elif cmd[0] == 'localCommand':
-            response = comm_inst._core._utils.localCommand(cmd[1])
+            response = localcommand.local_command(comm_inst._core, cmd[1])
         elif cmd[0] == 'pex':
             for i in comm_inst.timers:
                 if i.timerFunction.__name__ == 'lookupAdders':
@@ -49,7 +50,7 @@ def handle_daemon_commands(comm_inst):
 
         if cmd[0] not in ('', None):
             if response != '':
-                comm_inst._core._utils.localCommand('queueResponseAdd/' + cmd[4], post=True, postData={'data': response})
+                localcommand.local_command(comm_inst._core, 'queueResponseAdd/' + cmd[4], post=True, postData={'data': response})
         response = ''
 
     comm_inst.decrementThreadCount('daemonCommands')

onionr/communicatorutils/lookupadders.py

@@ -18,6 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import logger
+from onionrutils import stringvalidators
 
 def lookup_new_peer_transports_with_communicator(comm_inst):
     logger.info('Looking up new addresses...')
@@ -39,7 +40,7 @@ def lookup_new_peer_transports_with_communicator(comm_inst):
         invalid = []
         for x in newPeers:
             x = x.strip()
-            if not comm_inst._core._utils.validateID(x) or x in comm_inst.newPeers or x == comm_inst._core.hsAddress:
+            if not stringvalidators.validate_transport(x) or x in comm_inst.newPeers or x == comm_inst._core.hsAddress:
                 # avoid adding if its our address
                 invalid.append(x)
         for x in invalid:

onionr/communicatorutils/netcheck.py

@@ -20,17 +20,19 @@
 '''
 import logger
 from utils import netutils
+from onionrutils import localcommand
 def net_check(comm_inst):
     '''Check if we are connected to the internet or not when we can't connect to any peers'''
     rec = False # for detecting if we have received incoming connections recently
+    c = comm_inst._core
     if len(comm_inst.onlinePeers) == 0:
         try:
-            if (comm_inst._core._utils.getEpoch() - int(comm_inst._core._utils.localCommand('/lastconnect'))) <= 60:
+            if (c._utils.getEpoch() - int(localcommand.local_command(c, '/lastconnect'))) <= 60:
                 comm_inst.isOnline = True
                 rec = True
         except ValueError:
             pass
-        if not rec and not netutils.checkNetwork(comm_inst._core._utils, torPort=comm_inst.proxyPort):
+        if not rec and not netutils.checkNetwork(c._utils, torPort=comm_inst.proxyPort):
             if not comm_inst.shutdown:
                 logger.warn('Network check failed, are you connected to the Internet, and is Tor working?')
             comm_inst.isOnline = False

onionr/communicatorutils/servicecreator.py

@@ -18,6 +18,8 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import communicator, onionrblockapi
+from onionrutils import stringvalidators
+
 def service_creator(daemon):
     assert isinstance(daemon, communicator.OnionrCommunicatorDaemon)
     core = daemon._core
@@ -30,7 +32,7 @@ def service_creator(daemon):
         if not b in daemon.active_services:
             bl = onionrblockapi.Block(b, core=core, decrypt=True)
             bs = utils.bytesToStr(bl.bcontent) + '.onion'
-            if utils.validatePubKey(bl.signer) and utils.validateID(bs):
+            if utils.validatePubKey(bl.signer) and stringvalidators.validate_transport(bs):
                 signer = utils.bytesToStr(bl.signer)
                 daemon.active_services.append(b)
                 daemon.active_services.append(signer)

onionr/communicatorutils/uploadblocks.py

@@ -20,6 +20,7 @@
 import logger
 from communicatorutils import proxypicker
 import onionrblockapi as block
+from onionrutils import localcommand
 
 def upload_blocks_from_communicator(comm_inst):
     # when inserting a block, we try to upload it to a few peers to add some deniability
@@ -42,7 +43,7 @@ def upload_blocks_from_communicator(comm_inst):
                 proxyType = proxypicker.pick_proxy(peer)
                 logger.info("Uploading block to " + peer)
                 if not comm_inst._core._utils.doPostRequest(url, data=data, proxyType=proxyType) == False:
-                    comm_inst._core._utils.localCommand('waitforshare/' + bl, post=True)
+                    localcommand.local_command(comm_inst._core, 'waitforshare/' + bl, post=True)
                     finishedUploads.append(bl)
     for x in finishedUploads:
         try:

onionr/core.py

@@ -28,6 +28,7 @@ from onionrusers import onionrusers
 from onionrstorage import removeblock, setdata
 import dbcreator, onionrstorage, serializeddata, subprocesspow
 from etc import onionrvalues, powchoice
+from onionrutils import localcommand
 
 class Core:
     def __init__(self, torPort=0):
@@ -433,8 +434,8 @@ class Core:
                 retData = False
             else:
                 # Tell the api server through localCommand to wait for the daemon to upload this block to make statistical analysis more difficult
-                if self._utils.localCommand('/ping', maxWait=10) == 'pong!':
+                if localcommand.local_command(self, '/ping', maxWait=10) == 'pong!':
-                    self._utils.localCommand('/waitforshare/' + retData, post=True, maxWait=5)
+                    localcommand.local_command(self, '/waitforshare/' + retData, post=True, maxWait=5)
                     self.daemonQueueAdd('uploadBlock', retData)
                 self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
                 self._utils.processBlockMetadata(retData)
@@ -450,7 +451,7 @@ class Core:
         '''
             Introduces our node into the network by telling X many nodes our HS address
         '''
-        if self._utils.localCommand('/ping', maxWait=10) == 'pong!':
+        if localcommand.local_command(self, '/ping', maxWait=10) == 'pong!':
             self.daemonQueueAdd('announceNode')
             logger.info('Introduction command will be processed.', terminal=True)
         else:

onionr/coredb/daemonqueue/__init__.py

@@ -1,5 +1,7 @@
 import sqlite3, os
 import onionrevents as events
+from onionrutils import localcommand
+
 def daemon_queue(core_inst):
     '''
         Gives commands to the communication proccess/daemon by reading an sqlite3 database
@@ -55,7 +57,7 @@ def daemon_queue_get_response(core_inst, responseID=''):
         Get a response sent by communicator to the API, by requesting to the API
     '''
     assert len(responseID) > 0
-    resp = core_inst._utils.localCommand('queueResponse/' + responseID)
+    resp = localcommand.local_command(core_inst, 'queueResponse/' + responseID)
     return resp
 
 def clear_daemon_queue(core_inst):

onionr/coredb/keydb/addkeys.py

@@ -1,5 +1,7 @@
 import sqlite3
-import onionrevents as events, config
+import onionrevents as events
+from onionrutils import stringvalidators
+
 def add_peer(core_inst, peerID, name=''):
     '''
         Adds a public key to the key database (misleading function name)
@@ -40,8 +42,8 @@ def add_address(core_inst, address):
 
     if type(address) is None or len(address) == 0:
         return False
-    if core_inst._utils.validateID(address):
+    if stringvalidators.validate_transport(address):
-        if address == config.get('i2p.ownAddr', None) or address == core_inst.hsAddress:
+        if address == core_inst.config.get('i2p.ownAddr', None) or address == core_inst.hsAddress:
             return False
         conn = sqlite3.connect(core_inst.addressDB, timeout=30)
         c = conn.cursor()

onionr/coredb/keydb/removekeys.py

@@ -1,11 +1,13 @@
 import sqlite3
 import onionrevents as events
+from onionrutils import stringvalidators
+
 def remove_address(core_inst, address):
     '''
         Remove an address from the address database
     '''
 
-    if core_inst._utils.validateID(address):
+    if stringvalidators.validate_transport(address):
         conn = sqlite3.connect(core_inst.addressDB, timeout=30)
         c = conn.cursor()
         t = (address,)

onionr/httpapi/miscpublicapi/announce.py

@@ -21,6 +21,8 @@ import base64
 from flask import Response
 import logger
 from etc import onionrvalues
+from onionrutils import stringvalidators
+
 def handle_announce(clientAPI, request):
     '''
     accept announcement posts, validating POW
@@ -52,7 +54,7 @@ def handle_announce(clientAPI, request):
                 pass
             if powHash.startswith('0' * onionrvalues.OnionrValues().announce_pow):
                 newNode = clientAPI._core._utils.bytesToStr(newNode)
-                if clientAPI._core._utils.validateID(newNode) and not newNode in clientAPI._core.onionrInst.communicatorInst.newPeers:
+                if stringvalidators.validate_transport(newNode) and not newNode in clientAPI._core.onionrInst.communicatorInst.newPeers:
                     clientAPI._core.onionrInst.communicatorInst.newPeers.append(newNode)
                     resp = 'Success'
             else:

onionr/onionrcommands/__init__.py

@@ -22,6 +22,7 @@ import webbrowser, sys
 import logger
 from . import pubkeymanager, onionrstatistics, daemonlaunch, filecommands, plugincommands, keyadders
 from . import banblocks, exportblocks, openwebinterface, resettor
+from onionrutils import importnewblocks
 
 def show_help(o_inst, command):
 
@@ -110,8 +111,8 @@ def get_commands(onionr_inst):
     'listconn': onionr_inst.listConn,
     'list-conn': onionr_inst.listConn,
 
-    'import-blocks': onionr_inst.onionrUtils.importNewBlocks,
+    'import-blocks': importnewblocks.import_new_blocks,
-    'importblocks': onionr_inst.onionrUtils.importNewBlocks,
+    'importblocks': importnewblocks.import_new_blocks,
 
     'introduce': onionr_inst.onionrCore.introduceNode,
     'pex': onionr_inst.doPEX,

onionr/onionrcommands/daemonlaunch.py

@@ -23,9 +23,10 @@ from threading import Thread
 import onionr, api, logger, communicator
 import onionrevents as events
 from netcontroller import NetController
+from onionrutils import localcommand
 
 def _proper_shutdown(o_inst):
-    o_inst.onionrUtils.localCommand('shutdown')
+    localcommand.local_command(o_inst.onionrCore, 'shutdown')
     sys.exit(1)
 
 def daemon(o_inst):
@@ -63,7 +64,7 @@ def daemon(o_inst):
     net = NetController(o_inst.onionrCore.config.get('client.public.port', 59497), apiServerIP=apiHost)
     logger.info('Tor is starting...', terminal=True)
     if not net.startTor():
-        o_inst.onionrUtils.localCommand('shutdown')
+        localcommand.local_command(o_inst.onionrCore, 'shutdown')
         sys.exit(1)
     if len(net.myID) > 0 and o_inst.onionrCore.config.get('general.security_level', 1) == 0:
         logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID))
@@ -103,10 +104,10 @@ def daemon(o_inst):
 
     signal.signal(signal.SIGINT, _ignore_sigint)
     o_inst.onionrCore.daemonQueueAdd('shutdown')
-    o_inst.onionrUtils.localCommand('shutdown')
+    localcommand.local_command(o_inst.onionrCore, 'shutdown')
 
     net.killTor()
-    time.sleep(8) # Time to allow threads to finish, if not any "daemon" threads will be slaughtered http://docs.python.org/library/threading.html#threading.Thread.daemon
+    time.sleep(5) # Time to allow threads to finish, if not any "daemon" threads will be slaughtered http://docs.python.org/library/threading.html#threading.Thread.daemon
     o_inst.deleteRunFiles()
     return
 

onionr/onionrcommands/onionrstatistics.py

@@ -21,6 +21,7 @@ import os, uuid, time
 import logger, onionrutils
 from onionrblockapi import Block
 import onionr
+from onionrutils import checkcommunicator
 
 def show_stats(o_inst):
     try:
@@ -29,7 +30,7 @@ def show_stats(o_inst):
         signedBlocks = len(Block.getBlocks(signed = True))
         messages = {
             # info about local client
-            'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if o_inst.onionrUtils.isCommunicatorRunning(timeout = 9) else logger.colors.fg.red + 'Offline'),
+            'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if checkcommunicator.is_communicator_running(o_inst.onionrCore, timeout = 9) else logger.colors.fg.red + 'Offline'),
 
             # file and folder size stats
             'div1' : True, # this creates a solid line across the screen, a div

onionr/onionrcommands/openwebinterface.py

@@ -19,9 +19,10 @@
 '''
 import webbrowser
 import logger
+from onionrutils import getclientapiserver
 def open_home(o_inst):
     try:
-        url = o_inst.onionrUtils.getClientAPIServer()
+        url = getclientapiserver.get_client_API_server(o_inst.onionrCore)
     except FileNotFoundError:
         logger.error('Onionr seems to not be running (could not get api host)', terminal=True)
     else:

onionr/onionrcommands/resettor.py

@@ -19,11 +19,13 @@
 '''
 import os, shutil
 import logger, core
+from onionrutils import localcommand
+
 def reset_tor():
     c = core.Core()
     tor_dir = c.dataDir + 'tordata'
     if os.path.exists(tor_dir):
-        if c._utils.localCommand('/ping') == 'pong!':
+        if localcommand.local_command(c, '/ping') == 'pong!':
             logger.warn('Cannot delete Tor data while Onionr is running', terminal=True)
         else:
             shutil.rmtree(tor_dir)

onionr/onionrpluginapi.py

@@ -19,6 +19,7 @@
 '''
 
 import onionrplugins, core as onionrcore, logger
+from onionrutils import localcommand
 
 class DaemonAPI:
     def __init__(self, pluginapi):
@@ -40,7 +41,7 @@ class DaemonAPI:
         return
 
     def local_command(self, command):
-        return self.pluginapi.get_utils().localCommand(self, command)
+        return localcommand.local_command(self.pluginapi.get_core(), command)
 
     def queue_pop(self):
         return self.get_core().daemonQueue()

onionr/onionrservices/__init__.py

@@ -21,6 +21,7 @@ import time
 import stem
 import core
 from . import connectionserver, bootstrapservice
+from onionrutils import stringvalidators
 
 class OnionrServices:
     '''
@@ -39,7 +40,7 @@ class OnionrServices:
             When a client wants to connect, contact their bootstrap address and tell them our
             ephemeral address for our service by creating a new ConnectionServer instance
         '''
-        assert self._core._utils.validateID(address)
+        assert stringvalidators.validate_transport(address)
         BOOTSTRAP_TRIES = 10 # How many times to attempt contacting the bootstrap server
         TRY_WAIT = 3 # Seconds to wait before trying bootstrap again
         # HTTP is fine because .onion/i2p is encrypted/authenticated

onionr/onionrservices/bootstrapservice.py

@@ -24,6 +24,7 @@ from flask import Flask, Response
 import core
 from netcontroller import getOpenPort
 from . import httpheaders
+from onionrutils import stringvalidators
 
 def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300):
     '''
@@ -61,7 +62,7 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300):
 
     @bootstrap_app.route('/bs/<address>', methods=['POST'])
     def get_bootstrap(address):
-        if core_inst._utils.validateID(address + '.onion'):
+        if stringvalidators.validate_transport(address + '.onion'):
             # Set the bootstrap address then close the server
             bootstrap_address = address + '.onion'
             core_inst.keyStore.put(bs_id, bootstrap_address)

onionr/onionrutils/__init__.py

@@ -29,6 +29,7 @@ import storagecounter
 from etc import pgpwords, onionrvalues
 from onionrusers import onionrusers 
 from . import localcommand, blockmetadata, validatemetadata, basicrequests
+from . import stringvalidators
 
 config.reload()
 class OnionrUtils:
@@ -50,23 +51,6 @@ class OnionrUtils:
         '''
         epoch = self.getEpoch()
         return epoch - (epoch % roundS)
-    
-    def getClientAPIServer(self):
-        retData = ''
-        try:
-            with open(self._core.privateApiHostFile, 'r') as host:
-                hostname = host.read()
-        except FileNotFoundError:
-            raise FileNotFoundError
-        else:
-            retData += '%s:%s' % (hostname, config.get('client.client.port'))
-        return retData
-
-    def localCommand(self, command, data='', silent = True, post=False, postData = {}, maxWait=20):
-        '''
-            Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers.
-        '''
-        return localcommand.local_command(self, command, data, silent, post, postData, maxWait)
 
     def getHumanReadableID(self, pub=''):
         '''gets a human readable ID from a public key'''
@@ -132,19 +116,7 @@ class OnionrUtils:
         '''
             Validate if a string is a valid hash hex digest (does not compare, just checks length and charset)
         '''
-        retVal = True
+        return stringvalidators.validate_hash(self, data, length)
-        if data == False or data == True:
-            return False
-        data = data.strip()
-        if len(data) != length:
-            retVal = False
-        else:
-            try:
-                int(data, 16)
-            except ValueError:
-                retVal = False
-
-        return retVal
 
     def validateMetadata(self, metadata, blockData):
         '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
@@ -154,129 +126,7 @@ class OnionrUtils:
         '''
             Validate if a string is a valid base32 encoded Ed25519 key
         '''
-        if type(key) is type(None):
+        return stringvalidators.validate_pub_key(self, key)
-            return False
-        # Accept keys that have no = padding
-        key = unpaddedbase32.repad(self.strToBytes(key))
-
-        retVal = False
-        try:
-            nacl.signing.SigningKey(seed=key, encoder=nacl.encoding.Base32Encoder)
-        except nacl.exceptions.ValueError:
-            pass
-        except base64.binascii.Error as err:
-            pass
-        else:
-            retVal = True
-        return retVal
-
-    @staticmethod
-    def validateID(id):
-        '''
-            Validate if an address is a valid tor or i2p hidden service
-        '''
-        try:
-            idLength = len(id)
-            retVal = True
-            idNoDomain = ''
-            peerType = ''
-            # i2p b32 addresses are 60 characters long (including .b32.i2p)
-            if idLength == 60:
-                peerType = 'i2p'
-                if not id.endswith('.b32.i2p'):
-                    retVal = False
-                else:
-                    idNoDomain = id.split('.b32.i2p')[0]
-            # Onion v2's are 22 (including .onion), v3's are 62 with .onion
-            elif idLength == 22 or idLength == 62:
-                peerType = 'onion'
-                if not id.endswith('.onion'):
-                    retVal = False
-                else:
-                    idNoDomain = id.split('.onion')[0]
-            else:
-                retVal = False
-            if retVal:
-                if peerType == 'i2p':
-                    try:
-                        id.split('.b32.i2p')[2]
-                    except:
-                        pass
-                    else:
-                        retVal = False
-                elif peerType == 'onion':
-                    try:
-                        id.split('.onion')[2]
-                    except:
-                        pass
-                    else:
-                        retVal = False
-                if not idNoDomain.isalnum():
-                    retVal = False
-
-                # Validate address is valid base32 (when capitalized and minus extension); v2/v3 onions and .b32.i2p use base32
-                for x in idNoDomain.upper():
-                    if x not in string.ascii_uppercase and x not in '234567':
-                        retVal = False
-
-            return retVal
-        except:
-            return False
-
-    @staticmethod
-    def isIntegerString(data):
-        '''Check if a string is a valid base10 integer (also returns true if already an int)'''
-        try:
-            int(data)
-        except (ValueError, TypeError) as e:
-            return False
-        else:
-            return True
-
-    def isCommunicatorRunning(self, timeout = 5, interval = 0.1):
-        try:
-            runcheck_file = self._core.dataDir + '.runcheck'
-
-            if not os.path.isfile(runcheck_file):
-                open(runcheck_file, 'w+').close()
-
-            # self._core.daemonQueueAdd('runCheck') # deprecated
-            starttime = time.time()
-
-            while True:
-                time.sleep(interval)
-
-                if not os.path.isfile(runcheck_file):
-                    return True
-                elif time.time() - starttime >= timeout:
-                    return False
-        except:
-            return False
-
-    def importNewBlocks(self, scanDir=''):
-        '''
-            This function is intended to scan for new blocks ON THE DISK and import them
-        '''
-        blockList = self._core.getBlockList()
-        exist = False
-        if scanDir == '':
-            scanDir = self._core.blockDataLocation
-        if not scanDir.endswith('/'):
-            scanDir += '/'
-        for block in glob.glob(scanDir + "*.dat"):
-            if block.replace(scanDir, '').replace('.dat', '') not in blockList:
-                exist = True
-                logger.info('Found new block on dist %s' % block)
-                with open(block, 'rb') as newBlock:
-                    block = block.replace(scanDir, '').replace('.dat', '')
-                    if self._core._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''):
-                        self._core.addToBlockDB(block.replace('.dat', ''), dataSaved=True)
-                        logger.info('Imported block %s.' % block)
-                        self._core._utils.processBlockMetadata(block)
-                    else:
-                        logger.warn('Failed to verify hash for %s' % block)
-        if not exist:
-            logger.info('No blocks found to import')
 
     def getEpoch(self):
         '''returns epoch'''

onionr/onionrutils/checkcommunicator.py

@@ -0,0 +1,19 @@
+import time, os
+def is_communicator_running(core_inst, timeout = 5, interval = 0.1):
+    try:
+        runcheck_file = core_inst.dataDir + '.runcheck'
+
+        if not os.path.isfile(runcheck_file):
+            open(runcheck_file, 'w+').close()
+
+        starttime = time.time()
+
+        while True:
+            time.sleep(interval)
+
+            if not os.path.isfile(runcheck_file):
+                return True
+            elif time.time() - starttime >= timeout:
+                return False
+    except:
+        return False

onionr/onionrutils/getclientapiserver.py

@@ -0,0 +1,10 @@
+def get_client_API_server(core_inst):
+    retData = ''
+    try:
+        with open(core_inst.privateApiHostFile, 'r') as host:
+            hostname = host.read()
+    except FileNotFoundError:
+        raise FileNotFoundError
+    else:
+        retData += '%s:%s' % (hostname, core_inst.config.get('client.client.port'))
+    return retData

onionr/onionrutils/importnewblocks.py

@@ -0,0 +1,28 @@
+import glob
+import logger, core
+def import_new_blocks(core_inst=None, scanDir=''):
+    '''
+        This function is intended to scan for new blocks ON THE DISK and import them
+    '''
+    if core_inst is None:
+        core_inst = core.Core()
+    blockList = core_inst.getBlockList()
+    exist = False
+    if scanDir == '':
+        scanDir = core_inst.blockDataLocation
+    if not scanDir.endswith('/'):
+        scanDir += '/'
+    for block in glob.glob(scanDir + "*.dat"):
+        if block.replace(scanDir, '').replace('.dat', '') not in blockList:
+            exist = True
+            logger.info('Found new block on dist %s' % block)
+            with open(block, 'rb') as newBlock:
+                block = block.replace(scanDir, '').replace('.dat', '')
+                if core_inst._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''):
+                    core_inst.addToBlockDB(block.replace('.dat', ''), dataSaved=True)
+                    logger.info('Imported block %s.' % block)
+                    core_inst._utils.processBlockMetadata(block)
+                else:
+                    logger.warn('Failed to verify hash for %s' % block)
+    if not exist:
+        logger.info('No blocks found to import')

onionr/onionrutils/localcommand.py

@@ -1,6 +1,7 @@
 import urllib, requests, time
 import logger
-def local_command(utils_inst, command, data='', silent = True, post=False, postData = {}, maxWait=20):
+from onionrutils import getclientapiserver
+def local_command(core_inst, command, data='', silent = True, post=False, postData = {}, maxWait=20):
     '''
         Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers.
     '''
@@ -9,7 +10,7 @@ def local_command(utils_inst, command, data='', silent = True, post=False, postD
     waited = 0
     while hostname == '':
         try:
-            hostname = utils_inst.getClientAPIServer()
+            hostname = getclientapiserver.get_client_API_server(core_inst)
         except FileNotFoundError:
             time.sleep(1)
             waited += 1

onionr/onionrutils/stringvalidators.py

@@ -0,0 +1,97 @@
+import base64, string
+import unpaddedbase32, nacl.signing, nacl.encoding
+def validate_hash(utils_inst, data, length=64):
+    '''
+        Validate if a string is a valid hash hex digest (does not compare, just checks length and charset)
+    '''
+    retVal = True
+    if data == False or data == True:
+        return False
+    data = data.strip()
+    if len(data) != length:
+        retVal = False
+    else:
+        try:
+            int(data, 16)
+        except ValueError:
+            retVal = False
+
+    return retVal
+
+def validate_pub_key(utils_inst, key):
+    '''
+        Validate if a string is a valid base32 encoded Ed25519 key
+    '''
+    if type(key) is type(None):
+        return False
+    # Accept keys that have no = padding
+    key = unpaddedbase32.repad(utils_inst.strToBytes(key))
+
+    retVal = False
+    try:
+        nacl.signing.SigningKey(seed=key, encoder=nacl.encoding.Base32Encoder)
+    except nacl.exceptions.ValueError:
+        pass
+    except base64.binascii.Error as err:
+        pass
+    else:
+        retVal = True
+    return retVal
+
+def validate_transport(id):
+    try:
+        idLength = len(id)
+        retVal = True
+        idNoDomain = ''
+        peerType = ''
+        # i2p b32 addresses are 60 characters long (including .b32.i2p)
+        if idLength == 60:
+            peerType = 'i2p'
+            if not id.endswith('.b32.i2p'):
+                retVal = False
+            else:
+                idNoDomain = id.split('.b32.i2p')[0]
+        # Onion v2's are 22 (including .onion), v3's are 62 with .onion
+        elif idLength == 22 or idLength == 62:
+            peerType = 'onion'
+            if not id.endswith('.onion'):
+                retVal = False
+            else:
+                idNoDomain = id.split('.onion')[0]
+        else:
+            retVal = False
+        if retVal:
+            if peerType == 'i2p':
+                try:
+                    id.split('.b32.i2p')[2]
+                except:
+                    pass
+                else:
+                    retVal = False
+            elif peerType == 'onion':
+                try:
+                    id.split('.onion')[2]
+                except:
+                    pass
+                else:
+                    retVal = False
+            if not idNoDomain.isalnum():
+                retVal = False
+
+            # Validate address is valid base32 (when capitalized and minus extension); v2/v3 onions and .b32.i2p use base32
+            for x in idNoDomain.upper():
+                if x not in string.ascii_uppercase and x not in '234567':
+                    retVal = False
+
+        return retVal
+    except Exception as e:
+        return False
+
+def is_integer_string(data):
+    '''Check if a string is a valid base10 integer (also returns true if already an int)'''
+    try:
+        int(data)
+    except (ValueError, TypeError) as e:
+        return False
+    else:
+        return True

onionr/onionrutils/validatemetadata.py

@@ -1,6 +1,7 @@
 import json
 import logger, onionrexceptions
 from etc import onionrvalues
+from onionrutils import stringvalidators
 def validate_metadata(utils_inst, metadata, blockData):
     '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
     # TODO, make this check sane sizes
@@ -15,7 +16,7 @@ def validate_metadata(utils_inst, metadata, blockData):
             pass
 
     # Validate metadata dict for invalid keys to sizes that are too large
-    maxAge = utils_inst._coreconfig.get("general.max_block_age", onionrvalues.OnionrValues().default_expire)
+    maxAge = utils_inst._core.config.get("general.max_block_age", onionrvalues.OnionrValues().default_expire)
     if type(metadata) is dict:
         for i in metadata:
             try:
@@ -33,7 +34,7 @@ def validate_metadata(utils_inst, metadata, blockData):
                     logger.warn('Block metadata key ' + i + ' exceeded maximum size')
                     break
             if i == 'time':
-                if not utils_inst.isIntegerString(metadata[i]):
+                if not stringvalidators.is_integer_string(metadata[i]):
                     logger.warn('Block metadata time stamp is not integer string or int')
                     break
                 isFuture = (metadata[i] - utils_inst.getEpoch())

onionr/static-data/default-plugins/cliui/main.py

@@ -23,6 +23,7 @@ import threading, time, uuid, subprocess, sys
 import config, logger
 from onionrblockapi import Block
 import onionrplugins
+from onionrutils import localcommand
 
 plugin_name = 'cliui'
 PLUGIN_VERSION = '0.0.1'
@@ -48,7 +49,7 @@ class OnionrCLIUI:
     
     def isRunning(self):
         while not self.shutdown:
-            if self.myCore._utils.localCommand('ping', maxWait=5) == 'pong!':
+            if localcommand.local_command(self.myCore, 'ping', maxWait=5) == 'pong!':
                 self.running = 'Yes'
             else:
                 self.running = 'No'

onionr/static-data/default-plugins/esoteric/peerserver.py

@@ -19,6 +19,7 @@
 '''
 import sys, os, json
 import core
+from onionrutils import localcommand
 from flask import Response, request, redirect, Blueprint, abort, g
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
 direct_blueprint = Blueprint('esoteric', __name__)
@@ -47,9 +48,9 @@ def sendto():
         msg = ''
     else:
         msg = json.dumps(msg)
-        core_inst._utils.localCommand('/esoteric/addrec/%s' % (g.peer,), post=True, postData=msg)
+        localcommand.local_command(core_inst, '/esoteric/addrec/%s' % (g.peer,), post=True, postData=msg)
     return Response('success')
 
 @direct_blueprint.route('/esoteric/poll')
 def poll_chat():
-    return Response(core_inst._utils.localCommand('/esoteric/gets/%s' % (g.peer,)))
+    return Response(localcommand.local_command(core_inst, '/esoteric/gets/%s' % (g.peer,)))

onionr/static-data/default-plugins/pluginmanager/main.py

@@ -22,6 +22,7 @@
 import logger, config
 import os, sys, json, time, random, shutil, base64, getpass, datetime, re
 from onionrblockapi import Block
+from onionrutils import importnewblocks
 
 plugin_name = 'pluginmanager'
 
@@ -236,7 +237,7 @@ def pluginToBlock(plugin, import_block = True):
             # hash = pluginapi.get_core().insertBlock(, header = 'plugin', sign = True)
 
             if import_block:
-                pluginapi.get_utils().importNewBlocks()
+                importnewblocks.import_new_blocks(pluginapi.get_core())
 
             return hash
         else:

onionr/tests/test_stringvalidations.py

@@ -6,6 +6,7 @@ TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
 print("Test directory:", TEST_DIR)
 os.environ["ONIONR_HOME"] = TEST_DIR
 import core, onionr
+from onionrutils import stringvalidators
 
 core.Core()
 
@@ -13,7 +14,6 @@ class OnionrValidations(unittest.TestCase):
     
     def test_peer_validator(self):
         # Test hidden service domain validities
-        c = core.Core()
         valid = ['facebookcorewwwi.onion', 'vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd.onion', 
         '5bvb5ncnfr4dlsfriwczpzcvo65kn7fnnlnt2ln7qvhzna2xaldq.b32.i2p']
 
@@ -21,11 +21,11 @@ class OnionrValidations(unittest.TestCase):
 
         for x in valid:
             print('testing', x)
-            self.assertTrue(c._utils.validateID(x))
+            self.assertTrue(stringvalidators.validate_transport(x))
         
         for x in invalid:
             print('testing', x)
-            self.assertFalse(c._utils.validateID(x))
+            self.assertFalse(stringvalidators.validate_transport(x))
     
     def test_pubkey_validator(self):
         # Test ed25519 public key validity
@@ -44,14 +44,13 @@ class OnionrValidations(unittest.TestCase):
     def test_integer_string(self):
         valid = ["1", "100", 100, "-5", -5]
         invalid = ['test', "1d3434", "1e100", None]
-        c = core.Core()
 
         for x in valid:
             #print('testing', x)
-            self.assertTrue(c._utils.isIntegerString(x))   
+            self.assertTrue(stringvalidators.is_integer_string(x))   
         
         for x in invalid:
             #print('testing', x)
-            self.assertFalse(c._utils.isIntegerString(x))
+            self.assertFalse(stringvalidators.is_integer_string(x))
     
 unittest.main()