more progress in removing onionrutils class

onionr/api.py

@@ -23,8 +23,7 @@ from gevent import Timeout
 import flask
 from flask import request, Response, abort, send_from_directory
 import core
-from onionrblockapi import Block
+import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionrblockapi
-import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config
 import httpapi
 from httpapi import friendsapi, profilesapi, configapi, miscpublicapi
 from onionrservices import httpheaders
@@ -337,7 +336,7 @@ class API:
             resp = ''
             if self._core._utils.validateHash(name):
                 try:
-                    resp = Block(name, decrypt=True).bcontent
+                    resp = onionrblockapi.Block(name, decrypt=True).bcontent
                 except TypeError:
                     pass
             else:
@@ -374,7 +373,7 @@ class API:
             resp = 'Not Found'
             if self._core._utils.validateHash(bHash):
                 try:
-                    resp = Block(bHash).bcontent
+                    resp = onionrblockapi.Block(bHash).bcontent
                 except onionrexceptions.NoDataAvailable:
                     abort(404)
                 except TypeError:
@@ -505,7 +504,7 @@ class API:
     
     def getBlockData(self, bHash, decrypt=False, raw=False, headerOnly=False):
         assert self._core._utils.validateHash(bHash)
-        bl = Block(bHash, core=self._core)
+        bl = onionrblockapi.Block(bHash, core=self._core)
         if decrypt:
             bl.decrypt()
             if bl.isEncrypted and not bl.decrypted:
@@ -521,8 +520,8 @@ class API:
                         pass
             else:
                 validSig = False
-                signer = self._core._utils.bytesToStr(bl.signer)
+                signer = onionrutils.bytes_to_str(bl.signer)
-                if bl.isSigned() and self._core._utils.validatePubKey(signer) and bl.isSigner(signer):
+                if bl.isSigned() and onionrutils.stringvalidators.validate_pub_key(signer) and bl.isSigner(signer):
                     validSig = True                    
                 bl.bheader['validSig'] = validSig
                 bl.bheader['meta'] = ''

onionr/blockimporter.py

@@ -18,6 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import core, onionrexceptions, logger
+from onionrutils import validatemetadata, blockmetadata
 def importBlockFromData(content, coreInst):
     retData = False
 
@@ -34,17 +35,17 @@ def importBlockFromData(content, coreInst):
     except AttributeError:
         pass
 
-    metas = coreInst._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata
+    metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
     metadata = metas[0]
-    if coreInst._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid
+    if validatemetadata(metadata, metas[2]): # check if metadata is valid
         if coreInst._crypto.verifyPow(content): # check if POW is enough/correct
-            logger.info('Block passed proof, saving.')
+            logger.info('Block passed proof, saving.', terminal=True)
             try:
                 blockHash = coreInst.setData(content)
             except onionrexceptions.DiskAllocationReached:
                 pass
             else:
                 coreInst.addToBlockDB(blockHash, dataSaved=True)
-                coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
+                blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
                 retData = True
     return retData

onionr/communicatorutils/downloadblocks.py

@@ -19,6 +19,7 @@
 '''
 import communicator, onionrexceptions
 import logger, onionrpeers
+from onionrutils import blockmetadata
 
 def download_blocks_from_communicator(comm_inst):
     assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon)
@@ -72,7 +73,7 @@ def download_blocks_from_communicator(comm_inst):
                 pass
             if realHash == blockHash:
                 content = content.decode() # decode here because sha3Hash needs bytes above
-                metas = comm_inst._core._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata
+                metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
                 metadata = metas[0]
                 if comm_inst._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
                     if comm_inst._core._crypto.verifyPow(content): # check if POW is enough/correct
@@ -84,7 +85,7 @@ def download_blocks_from_communicator(comm_inst):
                             removeFromQueue = False
                         else:
                             comm_inst._core.addToBlockDB(blockHash, dataSaved=True)
-                            comm_inst._core._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
+                            blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
                     else:
                         logger.warn('POW failed for block %s.' % blockHash)
                 else:

onionr/communicatorutils/servicecreator.py

@@ -32,7 +32,7 @@ def service_creator(daemon):
         if not b in daemon.active_services:
             bl = onionrblockapi.Block(b, core=core, decrypt=True)
             bs = utils.bytesToStr(bl.bcontent) + '.onion'
-            if utils.validatePubKey(bl.signer) and stringvalidators.validate_transport(bs):
+            if stringvalidators.validate_pub_key(bl.signer) and stringvalidators.validate_transport(bs):
                 signer = utils.bytesToStr(bl.signer)
                 daemon.active_services.append(b)
                 daemon.active_services.append(signer)

onionr/core.py

@@ -28,7 +28,8 @@ from onionrusers import onionrusers
 from onionrstorage import removeblock, setdata
 import dbcreator, onionrstorage, serializeddata, subprocesspow
 from etc import onionrvalues, powchoice
-from onionrutils import localcommand
+from onionrutils import localcommand, stringvalidators, bytesconverter, epoch
+from onionrutils import blockmetadata
 
 class Core:
     def __init__(self, torPort=0):
@@ -320,9 +321,9 @@ class Core:
         if type(data) is None:
             raise ValueError('Data cannot be none')
 
-        createTime = self._utils.getRoundedEpoch()
+        createTime = epoch.get_epoch()
 
-        dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data))
+        dataNonce = bytesconverter.bytes_to_str(self._crypto.sha3Hash(data))
         try:
             with open(self.dataNonceFile, 'r') as nonces:
                 if dataNonce in nonces:
@@ -395,7 +396,7 @@ class Core:
             signature = self._crypto.symmetricEncrypt(signature, key=symKey, returnEncoded=True).decode()
             signer = self._crypto.symmetricEncrypt(signer, key=symKey, returnEncoded=True).decode()
         elif encryptType == 'asym':
-            if self._utils.validatePubKey(asymPeer):
+            if stringvalidators.validate_pub_key(asymPeer):
                 # Encrypt block data with forward secrecy key first, but not meta
                 jsonMeta = json.dumps(meta)
                 jsonMeta = self._crypto.pubKeyEncrypt(jsonMeta, asymPeer, encodedData=True).decode()
@@ -438,13 +439,13 @@ class Core:
                     localcommand.local_command(self, '/waitforshare/' + retData, post=True, maxWait=5)
                     self.daemonQueueAdd('uploadBlock', retData)
                 self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
-                self._utils.processBlockMetadata(retData)
+                blockmetadata.process_block_metadata(retData)
 
         if retData != False:
             if plaintextPeer == onionrvalues.DENIABLE_PEER_ADDRESS:
-                events.event('insertdeniable', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': self._utils.bytesToStr(asymPeer)}, onionr = self.onionrInst, threaded = True)
+                events.event('insertdeniable', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': bytesconverter.bytes_to_str(asymPeer)}, onionr = self.onionrInst, threaded = True)
             else:
-                events.event('insertblock', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': self._utils.bytesToStr(asymPeer)}, onionr = self.onionrInst, threaded = True)
+                events.event('insertblock', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': bytesconverter.bytes_to_str(asymPeer)}, onionr = self.onionrInst, threaded = True)
         return retData
 
     def introduceNode(self):

onionr/coredb/blockmetadb/add.py

@@ -1,4 +1,5 @@
 import os, sqlite3
+import onionrutils
 def add_to_block_DB(core_inst, newHash, selfInsert=False, dataSaved=False):
     '''
         Add a hash value to the block db
@@ -8,7 +9,7 @@ def add_to_block_DB(core_inst, newHash, selfInsert=False, dataSaved=False):
 
     if not os.path.exists(core_inst.blockDB):
         raise Exception('Block db does not exist')
-    if core_inst._utils.hasBlock(newHash):
+    if onionrutils.has_block(core_inst, newHash):
         return
     conn = sqlite3.connect(core_inst.blockDB, timeout=30)
     c = conn.cursor()

onionr/coredb/keydb/addkeys.py

@@ -10,7 +10,7 @@ def add_peer(core_inst, peerID, name=''):
         raise ValueError("specified id is already known")
 
     # This function simply adds a peer to the DB
-    if not core_inst._utils.validatePubKey(peerID):
+    if not stringvalidators.validate_pub_key(peerID):
         return False
 
     events.event('pubkey_add', data = {'key': peerID}, onionr = core_inst.onionrInst)

onionr/httpapi/miscpublicapi/getblocks.py

@@ -18,12 +18,12 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 from flask import Response, abort
-import config
+import config, onionrutils
 def get_public_block_list(clientAPI, publicAPI, request):
     # Provide a list of our blocks, with a date offset
     dateAdjust = request.args.get('date')
     bList = clientAPI._core.getBlockList(dateRec=dateAdjust)
-    if config.get('general.hide_created_blocks', True):
+    if clientAPI._core.config.get('general.hide_created_blocks', True):
         for b in publicAPI.hideBlocks:
             if b in bList:
                 # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
@@ -34,14 +34,14 @@ def get_block_data(clientAPI, publicAPI, data):
     '''data is the block hash in hex'''
     resp = ''
     if clientAPI._utils.validateHash(data):
-        if not config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks:
+        if not clientAPI._core.config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks:
             if data in clientAPI._core.getBlockList():
                 block = clientAPI.getBlockData(data, raw=True)
                 try:
                     block = block.encode() # Encode in case data is binary
                 except AttributeError:
                     abort(404)
-                block = clientAPI._core._utils.strToBytes(block)
+                block = onionrutils.str_to_bytes(block)
                 resp = block
     if len(resp) == 0:
         abort(404)

onionr/netcontroller.py

@@ -20,7 +20,6 @@
 import subprocess, os, sys, time, signal, base64, socket
 from shutil import which
 import logger, config
-from onionrblockapi import Block
 config.reload()
 def getOpenPort():
     # taken from (but modified) https://stackoverflow.com/a/2838309 by https://stackoverflow.com/users/133374/albert ccy-by-sa-3 https://creativecommons.org/licenses/by-sa/3.0/

onionr/onionr.py

@@ -21,6 +21,10 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import sys
+ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net'
+ONIONR_VERSION = '0.0.0' # for debugging and stuff
+ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
+API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
 MIN_PY_VERSION = 6
 if sys.version_info[0] == 2 or sys.version_info[1] < MIN_PY_VERSION:
     sys.stderr.write('Error, Onionr requires Python 3.%s+' % (MIN_PY_VERSION,))
@@ -40,10 +44,6 @@ try:
 except ImportError:
     raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)")
 
-ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net'
-ONIONR_VERSION = '0.0.0' # for debugging and stuff
-ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
-API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
 
 class Onionr:
     def __init__(self):
@@ -72,7 +72,7 @@ class Onionr:
         data_exists = Onionr.setupConfig(self.dataDir, self)
 
         if netcontroller.torBinary() is None:
-            logger.error('Tor is not installed')
+            logger.error('Tor is not installed', terminal=True)
             sys.exit(1)
 
         # If block data folder does not exist
@@ -101,7 +101,6 @@ class Onionr:
         self.onionrCore = core.Core()
         self.onionrCore.onionrInst = self
         #self.deleteRunFiles()
-        self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore)
 
         self.clientAPIInst = '' # Client http api instance
         self.publicAPIInst = '' # Public http api instance

onionr/onionrblockapi.py

@@ -21,6 +21,7 @@
 import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions
 import json, os, sys, datetime, base64, onionrstorage
 from onionrusers import onionrusers
+from onionrutils import stringvalidators
 
 class Block:
     blockCacheOrder = list() # NEVER write your own code that writes to this!
@@ -441,7 +442,7 @@ class Block:
         '''
 
         try:
-            if (not self.isSigned()) or (not self.getCore()._utils.validatePubKey(signer)):
+            if (not self.isSigned()) or (not stringvalidators.validate_pub_key(signer)):
                 return False
 
             return bool(self.getCore()._crypto.edVerify(self.getSignedData(), signer, self.getSignature(), encodedData = encodedData))

onionr/onionrcommands/keyadders.py

@@ -25,7 +25,7 @@ def add_peer(o_inst):
     except IndexError:
         pass
     else:
-        if o_inst.onionrUtils.hasKey(newPeer):
+        if newPeer in o_inst.onionrCore.listPeers():
             logger.info('We already have that key', terminal=True)
             return
         logger.info("Adding peer: " + logger.colors.underline + newPeer, terminal=True)

onionr/onionrcommands/onionrstatistics.py

@@ -21,7 +21,7 @@ import os, uuid, time
 import logger, onionrutils
 from onionrblockapi import Block
 import onionr
-from onionrutils import checkcommunicator
+from onionrutils import checkcommunicator, mnemonickeys
 
 def show_stats(o_inst):
     try:
@@ -87,7 +87,7 @@ def show_details(o_inst):
         'Node Address' : o_inst.get_hostname(),
         'Web Password' : o_inst.getWebPassword(),
         'Public Key' : o_inst.onionrCore._crypto.pubKey,
-        'Human-readable Public Key' : o_inst.onionrCore._utils.getHumanReadableID()
+        'Human-readable Public Key' : mnemonickeys.get_human_readable_ID(o_inst.onionrCore)
     }
 
     for detail in details:

onionr/onionrcommands/pubkeymanager.py

@@ -20,6 +20,7 @@
 
 import sys, getpass
 import logger, onionrexceptions
+from onionrutils import stringvalidators
 from onionrusers import onionrusers, contactmanager
 import unpaddedbase32
 def add_ID(o_inst):
@@ -55,7 +56,7 @@ def change_ID(o_inst):
     except IndexError:
         logger.warn('Specify pubkey to use', terminal=True)
     else:
-        if o_inst.onionrUtils.validatePubKey(key):
+        if stringvalidators.validate_pub_key(key):
             if key in o_inst.onionrCore._crypto.keyManager.getPubkeyList():
                 o_inst.onionrCore.config.set('general.public_key', key)
                 o_inst.onionrCore.config.save()
@@ -82,7 +83,7 @@ def friend_command(o_inst):
         elif action in ('add', 'remove'):
             try:
                 friend = sys.argv[3]
-                if not o_inst.onionrUtils.validatePubKey(friend):
+                if not stringvalidators.validate_pub_key(friend):
                     raise onionrexceptions.InvalidPubkey('Public key is invalid')
                 if friend not in o_inst.onionrCore.listPeers():
                     raise onionrexceptions.KeyNotKnown

onionr/onionrcrypto.py

@@ -21,7 +21,8 @@ import os, binascii, base64, hashlib, time, sys, hmac, secrets
 import nacl.signing, nacl.encoding, nacl.public, nacl.hash, nacl.pwhash, nacl.utils, nacl.secret
 import unpaddedbase32
 import logger, onionrproofs
-import onionrexceptions, keymanager, core
+from onionrutils import stringvalidators
+import onionrexceptions, keymanager, core, onionrutils
 import config
 config.reload()
 
@@ -38,8 +39,8 @@ class OnionrCrypto:
 
         # Load our own pub/priv Ed25519 keys, gen & save them if they don't exist
         if os.path.exists(self._keyFile):
-            if len(config.get('general.public_key', '')) > 0:
+            if len(self._core.config.get('general.public_key', '')) > 0:
-                self.pubKey = config.get('general.public_key')
+                self.pubKey = self._core.config.get('general.public_key')
             else:
                 self.pubKey = self.keyManager.getPubkeyList()[0]
             self.privKey = self.keyManager.getPrivkey(self.pubKey)
@@ -94,10 +95,10 @@ class OnionrCrypto:
 
     def pubKeyEncrypt(self, data, pubkey, encodedData=False):
         '''Encrypt to a public key (Curve25519, taken from base32 Ed25519 pubkey)'''
-        pubkey = unpaddedbase32.repad(self._core._utils.strToBytes(pubkey))
+        pubkey = unpaddedbase32.repad(onionrutils.str_to_bytes(pubkey))
         retVal = ''
         box = None
-        data = self._core._utils.strToBytes(data)
+        data = onionrutils.str_to_bytes(data)
         
         pubkey = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_public_key()
 
@@ -122,7 +123,7 @@ class OnionrCrypto:
             privkey = self.privKey
         ownKey = nacl.signing.SigningKey(seed=privkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_private_key()
 
-        if self._core._utils.validatePubKey(privkey):
+        if stringvalidators.validate_pub_key(privkey):
             privkey = nacl.signing.SigningKey(seed=privkey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_private_key()
             anonBox = nacl.public.SealedBox(privkey)
         else:
@@ -181,7 +182,7 @@ class OnionrCrypto:
     def generateDeterministic(self, passphrase, bypassCheck=False):
         '''Generate a Ed25519 public key pair from a password'''
         passStrength = self.deterministicRequirement
-        passphrase = self._core._utils.strToBytes(passphrase) # Convert to bytes if not already
+        passphrase = onionrutils.str_to_bytes(passphrase) # Convert to bytes if not already
         # Validate passphrase length
         if not bypassCheck:
             if len(passphrase) < passStrength:
@@ -201,7 +202,7 @@ class OnionrCrypto:
         if pubkey == '':
             pubkey = self.pubKey
         prev = ''
-        pubkey = self._core._utils.strToBytes(pubkey)
+        pubkey = onionrutils.str_to_bytes(pubkey)
         for i in range(self.HASH_ID_ROUNDS):
             try:
                 prev = prev.encode()
@@ -250,8 +251,8 @@ class OnionrCrypto:
         
         difficulty = onionrproofs.getDifficultyForNewBlock(blockContent, ourBlock=False, coreInst=self._core)
         
-        if difficulty < int(config.get('general.minimum_block_pow')):
+        if difficulty < int(self._core.config.get('general.minimum_block_pow')):
-            difficulty = int(config.get('general.minimum_block_pow'))
+            difficulty = int(self._core.config.get('general.minimum_block_pow'))
         mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
         puzzle = mainHash[:difficulty]
 

onionr/onionrproofs.py

@@ -18,7 +18,8 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import multiprocessing, nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, sys, json
-import core, onionrutils, config, logger, onionrblockapi
+import core, config, logger, onionrblockapi
+from onionrutils import bytesconverter
 
 config.reload()
 
@@ -31,8 +32,6 @@ def getDifficultyModifier(coreOrUtilsInst=None):
     retData = 0
     if isinstance(classInst, core.Core):
         useFunc = classInst._utils.storageCounter.getPercent
-    elif isinstance(classInst, onionrutils.OnionrUtils):
-        useFunc = classInst.storageCounter.getPercent
     else:
         useFunc = core.Core()._utils.storageCounter.getPercent
 
@@ -56,7 +55,7 @@ def getDifficultyForNewBlock(data, ourBlock=True, coreInst=None):
     if isinstance(data, onionrblockapi.Block):
         dataSize = len(data.getRaw().encode('utf-8'))
     else:
-        dataSize = len(onionrutils.OnionrUtils.strToBytes(data))
+        dataSize = len(bytesconverter.str_to_bytes(data))
 
     if ourBlock:
         minDifficulty = config.get('general.minimum_send_pow', 4)

onionr/onionrservices/bootstrapservice.py

@@ -33,7 +33,7 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300):
     if core_inst is None:
         core_inst = core.Core()
     
-    if not core_inst._utils.validatePubKey(peer):
+    if not stringvalidators.validate_pub_key(peer):
         raise ValueError('Peer must be valid base32 ed25519 public key')
 
     bootstrap_port = getOpenPort()

onionr/onionrservices/connectionserver.py

@@ -24,6 +24,7 @@ import core, logger, httpapi
 import onionrexceptions
 from netcontroller import getOpenPort
 import api
+from onionrutils import stringvalidators
 from . import httpheaders
 
 class ConnectionServer:
@@ -33,7 +34,7 @@ class ConnectionServer:
         else:
             self.core_inst = core_inst
 
-        if not core_inst._utils.validatePubKey(peer):
+        if not stringvalidators.validate_pub_key(peer):
             raise ValueError('Peer must be valid base32 ed25519 public key')
         
         socks = core_inst.config.get('tor.socksport') # Load config for Tor socks port for proxy

onionr/onionrstorage/__init__.py

@@ -18,6 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 import core, sys, sqlite3, os, dbcreator, onionrexceptions
+from onionrutils import bytesconverter
 
 DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
 
@@ -82,7 +83,7 @@ def getData(coreInst, bHash):
     assert isinstance(coreInst, core.Core)
     assert coreInst._utils.validateHash(bHash)
 
-    bHash = coreInst._utils.bytesToStr(bHash)
+    bHash = bytesconverter.bytes_to_str(bHash)
 
     # First check DB for data entry by hash
     # if no entry, check disk

onionr/onionrusers/contactmanager.py

@@ -20,10 +20,11 @@
 import os, json, onionrexceptions
 import unpaddedbase32
 from onionrusers import onionrusers
+from onionrutils import bytesconverter
 
 class ContactManager(onionrusers.OnionrUser):
     def __init__(self, coreInst, publicKey, saveUser=False, recordExpireSeconds=5):
-        publicKey = unpaddedbase32.repad(coreInst._utils.strToBytes(publicKey)).decode()
+        publicKey = unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)).decode()
         super(ContactManager, self).__init__(coreInst, publicKey, saveUser=saveUser)
         self.dataDir = coreInst.dataDir + '/contacts/'
         self.dataFile = '%s/contacts/%s.json' % (coreInst.dataDir, publicKey)

onionr/onionrusers/onionrusers.py

@@ -17,7 +17,9 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import onionrblockapi, logger, onionrexceptions, json, sqlite3, time
+import logger, onionrexceptions, json, sqlite3, time
+from onionrutils import stringvalidators, bytesconverter
+
 import unpaddedbase32
 import nacl.exceptions
 
@@ -56,7 +58,7 @@ class OnionrUser:
             Takes an instance of onionr core, a base32 encoded ed25519 public key, and a bool saveUser
             saveUser determines if we should add a user to our peer database or not.
         '''
-        publicKey = unpaddedbase32.repad(coreInst._utils.strToBytes(publicKey)).decode()
+        publicKey = unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)).decode()
 
         self.trust = 0
         self._core = coreInst
@@ -103,7 +105,7 @@ class OnionrUser:
         deleteExpiredKeys(self._core)
         retData = ''
         forwardKey = self._getLatestForwardKey()
-        if self._core._utils.validatePubKey(forwardKey[0]):
+        if stringvalidators.validate_pub_key(forwardKey[0]):
             retData = self._core._crypto.pubKeyEncrypt(data, forwardKey[0], encodedData=True)
         else:
             raise onionrexceptions.InvalidPubkey("No valid forward secrecy key available for this user")
@@ -190,8 +192,8 @@ class OnionrUser:
         return list(keyList)
 
     def addForwardKey(self, newKey, expire=DEFAULT_KEY_EXPIRE):
-        newKey = self._core._utils.bytesToStr(unpaddedbase32.repad(self._core._utils.strToBytes(newKey)))
+        newKey = self._core._utils.bytesToStr(unpaddedbase32.repad(bytesconverter.str_to_bytes(newKey)))
-        if not self._core._utils.validatePubKey(newKey):
+        if not stringvalidators.validate_pub_key(newKey):
             # Do not add if something went wrong with the key
             raise onionrexceptions.InvalidPubkey(newKey)
 

onionr/onionrutils/__init__.py

@@ -22,13 +22,11 @@ import sys, os, sqlite3, binascii, time, base64, json, glob, shutil, math, re, u
 import requests
 import nacl.signing, nacl.encoding
 import unpaddedbase32
-from onionrblockapi import Block
 import onionrexceptions, config, logger
 import onionrevents
 import storagecounter
 from etc import pgpwords, onionrvalues
-from onionrusers import onionrusers 
+from . import localcommand, blockmetadata, basicrequests, validatemetadata
-from . import localcommand, blockmetadata, validatemetadata, basicrequests
 from . import stringvalidators
 
 config.reload()
@@ -45,39 +43,6 @@ class OnionrUtils:
         self.storageCounter = storagecounter.StorageCounter(self._core) # used to keep track of how much data onionr is using on disk
         return
 
-    def getRoundedEpoch(self, roundS=60):
-        '''
-            Returns the epoch, rounded down to given seconds (Default 60)
-        '''
-        epoch = self.getEpoch()
-        return epoch - (epoch % roundS)
-
-    def getHumanReadableID(self, pub=''):
-        '''gets a human readable ID from a public key'''
-        if pub == '':
-            pub = self._core._crypto.pubKey
-        pub = base64.b16encode(base64.b32decode(pub)).decode()
-        return ' '.join(pgpwords.wordify(pub))
-    
-    def convertHumanReadableID(self, pub):
-        '''Convert a human readable pubkey id to base32'''
-        pub = pub.lower()
-        return self.bytesToStr(base64.b32encode(binascii.unhexlify(pgpwords.hexify(pub.strip()))))
-
-    def getBlockMetadataFromData(self, blockData):
-        '''
-            accepts block contents as string, returns a tuple of 
-            metadata, meta (meta being internal metadata, which will be 
-            returned as an encrypted base64 string if it is encrypted, dict if not).
-        '''
-        return blockmetadata.get_block_metadata_from_data(self, blockData)
-
-    def processBlockMetadata(self, blockHash):
-        '''
-            Read metadata from a block and cache it to the block database
-        '''
-        return blockmetadata.process_block_metadata(self, blockHash)
-
     def escapeAnsi(self, line):
         '''
             Remove ANSI escape codes from a string with regex
@@ -88,46 +53,12 @@ class OnionrUtils:
         ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]')
         return ansi_escape.sub('', line)
 
-    def hasBlock(self, hash):
-        '''
-            Check for new block in the list
-        '''
-        conn = sqlite3.connect(self._core.blockDB)
-        c = conn.cursor()
-        if not self.validateHash(hash):
-            raise Exception("Invalid hash")
-        for result in c.execute("SELECT COUNT() FROM hashes WHERE hash = ?", (hash,)):
-            if result[0] >= 1:
-                conn.commit()
-                conn.close()
-                return True
-            else:
-                conn.commit()
-                conn.close()
-                return False
-
-    def hasKey(self, key):
-        '''
-            Check for key in list of public keys
-        '''
-        return key in self._core.listPeers()
-
     def validateHash(self, data, length=64):
         '''
             Validate if a string is a valid hash hex digest (does not compare, just checks length and charset)
         '''
         return stringvalidators.validate_hash(self, data, length)
 
-    def validateMetadata(self, metadata, blockData):
-        '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
-        return validatemetadata.validate_metadata(self, metadata, blockData)
-
-    def validatePubKey(self, key):
-        '''
-            Validate if a string is a valid base32 encoded Ed25519 key
-        '''
-        return stringvalidators.validate_pub_key(self, key)
-
     def getEpoch(self):
         '''returns epoch'''
         return math.floor(time.time())
@@ -144,21 +75,6 @@ class OnionrUtils:
         '''
         return basicrequests.do_get_request(self, url, port, proxyType, ignoreAPI, returnHeaders)
 
-    @staticmethod
-    def strToBytes(data):
-        try:
-            data = data.encode()
-        except AttributeError:
-            pass
-        return data
-    @staticmethod
-    def bytesToStr(data):
-        try:
-            data = data.decode()
-        except AttributeError:
-            pass
-        return data
-
 def size(path='.'):
     '''
         Returns the size of a folder's contents in bytes
@@ -183,4 +99,22 @@ def humanSize(num, suffix='B'):
         if abs(num) < 1024.0:
             return "%.1f %s%s" % (num, unit, suffix)
         num /= 1024.0
-    return "%.1f %s%s" % (num, 'Yi', suffix)
+    return "%.1f %s%s" % (num, 'Yi', suffix)
+
+def has_block(core_inst, hash):
+    '''
+        Check for new block in the list
+    '''
+    conn = sqlite3.connect(core_inst.blockDB)
+    c = conn.cursor()
+    if not stringvalidators.validate_hash(hash):
+        raise Exception("Invalid hash")
+    for result in c.execute("SELECT COUNT() FROM hashes WHERE hash = ?", (hash,)):
+        if result[0] >= 1:
+            conn.commit()
+            conn.close()
+            return True
+        else:
+            conn.commit()
+            conn.close()
+            return False

onionr/onionrutils/basicrequests.py

@@ -1,6 +1,5 @@
 import requests
 import logger, onionrexceptions
-from onionr import API_VERSION
 def do_post_request(utils_inst, url, data={}, port=0, proxyType='tor'):
     '''
     Do a POST request through a local tor or i2p instance
@@ -29,6 +28,7 @@ def do_get_request(utils_inst, url, port=0, proxyType='tor', ignoreAPI=False, re
     '''
     Do a get request through a local tor or i2p instance
     '''
+    API_VERSION = utils_inst._core.onionrInst.API_VERSION
     retData = False
     if proxyType == 'tor':
         if port == 0:

onionr/onionrutils/blockmetadata.py

@@ -2,8 +2,9 @@ import json
 import logger, onionrevents
 from onionrusers import onionrusers
 from etc import onionrvalues
-from onionrblockapi import Block
+import onionrblockapi
-def get_block_metadata_from_data(utils_inst, blockData):
+from . import epoch
+def get_block_metadata_from_data(blockData):
     '''
         accepts block contents as string, returns a tuple of 
         metadata, meta (meta being internal metadata, which will be 
@@ -36,8 +37,8 @@ def process_block_metadata(utils_inst, blockHash):
     '''
         Read metadata from a block and cache it to the block database
     '''
-    curTime = utils_inst.getRoundedEpoch(roundS=60)
+    curTime = epoch.get_rounded_epoch(roundS=60)
-    myBlock = Block(blockHash, utils_inst._core)
+    myBlock = onionrblockapi.Block(blockHash, utils_inst._core)
     if myBlock.isEncrypted:
         myBlock.decrypt()
     if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted):

onionr/onionrutils/bytesconverter.py

@@ -0,0 +1,13 @@
+def str_to_bytes(data):
+    try:
+        data = data.encode()
+    except AttributeError:
+        pass
+    return data
+
+def bytes_to_str(data):
+    try:
+        data = data.decode()
+    except AttributeError:
+        pass
+    return data

onionr/onionrutils/epoch.py

@@ -0,0 +1,11 @@
+import math, time
+def get_rounded_epoch(roundS=60):
+    '''
+        Returns the epoch, rounded down to given seconds (Default 60)
+    '''
+    epoch = get_epoch()
+    return epoch - (epoch % roundS)
+
+def get_epoch(self):
+    '''returns epoch'''
+    return math.floor(time.time())

onionr/onionrutils/importnewblocks.py

@@ -1,5 +1,6 @@
 import glob
 import logger, core
+from onionrutils import blockmetadata
 def import_new_blocks(core_inst=None, scanDir=''):
     '''
         This function is intended to scan for new blocks ON THE DISK and import them
@@ -21,7 +22,7 @@ def import_new_blocks(core_inst=None, scanDir=''):
                 if core_inst._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''):
                     core_inst.addToBlockDB(block.replace('.dat', ''), dataSaved=True)
                     logger.info('Imported block %s.' % block)
-                    core_inst._utils.processBlockMetadata(block)
+                    blockmetadata.process_block_metadata(block)
                 else:
                     logger.warn('Failed to verify hash for %s' % block)
     if not exist:

onionr/onionrutils/mnemonickeys.py

@@ -0,0 +1,8 @@
+import base64
+from etc import pgpwords
+def get_human_readable_ID(core_inst, pub=''):
+    '''gets a human readable ID from a public key'''
+    if pub == '':
+        pub = core_inst._crypto.pubKey
+    pub = base64.b16encode(base64.b32decode(pub)).decode()
+    return ' '.join(pgpwords.wordify(pub))

onionr/onionrutils/stringvalidators.py

@@ -1,4 +1,4 @@
-import base64, string
+import base64, string, onionrutils
 import unpaddedbase32, nacl.signing, nacl.encoding
 def validate_hash(utils_inst, data, length=64):
     '''
@@ -18,14 +18,14 @@ def validate_hash(utils_inst, data, length=64):
 
     return retVal
 
-def validate_pub_key(utils_inst, key):
+def validate_pub_key(key):
     '''
         Validate if a string is a valid base32 encoded Ed25519 key
     '''
     if type(key) is type(None):
         return False
     # Accept keys that have no = padding
-    key = unpaddedbase32.repad(utils_inst.strToBytes(key))
+    key = unpaddedbase32.repad(onionrutils.str_to_bytes(key))
 
     retVal = False
     try:

onionr/onionrutils/validatemetadata.py

@@ -2,7 +2,7 @@ import json
 import logger, onionrexceptions
 from etc import onionrvalues
 from onionrutils import stringvalidators
-def validate_metadata(utils_inst, metadata, blockData):
+def validate_metadata(core_inst, metadata, blockData):
     '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
     # TODO, make this check sane sizes
     retData = False
@@ -16,11 +16,11 @@ def validate_metadata(utils_inst, metadata, blockData):
             pass
 
     # Validate metadata dict for invalid keys to sizes that are too large
-    maxAge = utils_inst._core.config.get("general.max_block_age", onionrvalues.OnionrValues().default_expire)
+    maxAge = core_inst.config.get("general.max_block_age", onionrvalues.OnionrValues().default_expire)
     if type(metadata) is dict:
         for i in metadata:
             try:
-                utils_inst._core.requirements.blockMetadataLengths[i]
+                core_inst.requirements.blockMetadataLengths[i]
             except KeyError:
                 logger.warn('Block has invalid metadata key ' + i)
                 break
@@ -30,25 +30,25 @@ def validate_metadata(utils_inst, metadata, blockData):
                     testData = len(testData)
                 except (TypeError, AttributeError) as e:
                     testData = len(str(testData))
-                if utils_inst._core.requirements.blockMetadataLengths[i] < testData:
+                if core_inst.requirements.blockMetadataLengths[i] < testData:
                     logger.warn('Block metadata key ' + i + ' exceeded maximum size')
                     break
             if i == 'time':
                 if not stringvalidators.is_integer_string(metadata[i]):
                     logger.warn('Block metadata time stamp is not integer string or int')
                     break
-                isFuture = (metadata[i] - utils_inst.getEpoch())
+                isFuture = (metadata[i] - core_inst.getEpoch())
                 if isFuture > maxClockDifference:
                     logger.warn('Block timestamp is skewed to the future over the max %s: %s' (maxClockDifference, isFuture))
                     break
-                if (utils_inst.getEpoch() - metadata[i]) > maxAge:
+                if (core_inst.getEpoch() - metadata[i]) > maxAge:
                     logger.warn('Block is outdated: %s' % (metadata[i],))
                     break
             elif i == 'expire':
                 try:
-                    assert int(metadata[i]) > utils_inst.getEpoch()
+                    assert int(metadata[i]) > core_inst.getEpoch()
                 except AssertionError:
-                    logger.warn('Block is expired: %s less than %s' % (metadata[i], utils_inst.getEpoch()))
+                    logger.warn('Block is expired: %s less than %s' % (metadata[i], core_inst.getEpoch()))
                     break
             elif i == 'encryptType':
                 try:
@@ -59,9 +59,9 @@ def validate_metadata(utils_inst, metadata, blockData):
         else:
             # if metadata loop gets no errors, it does not break, therefore metadata is valid
             # make sure we do not have another block with the same data content (prevent data duplication and replay attacks)
-            nonce = utils_inst._core._utils.bytesToStr(utils_inst._core._crypto.sha3Hash(blockData))
+            nonce = core_inst._utils.bytesToStr(core_inst._crypto.sha3Hash(blockData))
             try:
-                with open(utils_inst._core.dataNonceFile, 'r') as nonceFile:
+                with open(core_inst.dataNonceFile, 'r') as nonceFile:
                     if nonce in nonceFile.read():
                         retData = False # we've seen that nonce before, so we can't pass metadata
                         raise onionrexceptions.DataExists

onionr/static-data/default-plugins/encrypt/main.py

@@ -21,6 +21,7 @@
 # Imports some useful libraries
 import logger, config, threading, time, datetime, sys, json
 from onionrblockapi import Block
+from onionrutils import stringvalidators
 import onionrexceptions, onionrusers
 import locale
 locale.setlocale(locale.LC_ALL, '')
@@ -43,7 +44,7 @@ class PlainEncryption:
             pass
 
         try:
-            if not self.api.get_core()._utils.validatePubKey(sys.argv[2]):
+            if not stringvalidators.validate_pub_key(sys.argv[2]):
                 raise onionrexceptions.InvalidPubkey
         except (ValueError, IndexError) as e:
             logger.error("Peer public key not specified", terminal=True)

onionr/static-data/default-plugins/esoteric/main.py

@@ -23,6 +23,7 @@ import locale, sys, os, threading, json
 locale.setlocale(locale.LC_ALL, '')
 import onionrservices, logger
 from onionrservices import bootstrapservice
+from onionrutils import stringvalidators
 
 plugin_name = 'esoteric'
 PLUGIN_VERSION = '0.0.0'
@@ -66,7 +67,7 @@ class Esoteric:
     def create(self):
         try:
             peer = sys.argv[2]
-            if not self.myCore._utils.validatePubKey(peer):
+            if not stringvalidators.validate_pub_key(peer):
                 exit_with_error('Invalid public key specified')
         except IndexError:
             exit_with_error('You must specify a peer public key')

onionr/static-data/default-plugins/metadataprocessor/main.py

@@ -23,6 +23,7 @@ import logger, config
 import os, sys, json, time, random, shutil, base64, getpass, datetime, re
 from onionrblockapi import Block
 import onionrusers, onionrexceptions
+from onionrutils import stringvalidators
 
 plugin_name = 'metadataprocessor'
 
@@ -36,7 +37,7 @@ def _processForwardKey(api, myBlock):
     key = myBlock.getMetadata('newFSKey')
 
     # We don't need to validate here probably, but it helps
-    if api.get_utils().validatePubKey(key):
+    if stringvalidators.validate_pub_key(key):
         peer.addForwardKey(key)
     else:
         raise onionrexceptions.InvalidPubkey("%s is not a valid pubkey key" % (key,))

onionr/static-data/default-plugins/pluginmanager/main.py

@@ -22,7 +22,7 @@
 import logger, config
 import os, sys, json, time, random, shutil, base64, getpass, datetime, re
 from onionrblockapi import Block
-from onionrutils import importnewblocks
+from onionrutils import importnewblocks, stringvalidators, 
 
 plugin_name = 'pluginmanager'
 
@@ -399,13 +399,13 @@ def commandInstallPlugin():
 
         valid_hash = pluginapi.get_utils().validateHash(pkobh)
         real_block = False
-        valid_key = pluginapi.get_utils().validatePubKey(pkobh)
+        valid_key = stringvalidators.validate_pub_key(pkobh)
         real_key = False
 
         if valid_hash:
             real_block = Block.exists(pkobh)
         elif valid_key:
-            real_key = pluginapi.get_utils().hasKey(pkobh)
+            real_key = pkobh in pluginapi.get_core().listPeers()
 
         blockhash = None
 
@@ -493,7 +493,7 @@ def commandAddRepository():
                     pluginslist = dict()
 
                     for pluginname, distributor in blockContent['plugins']:
-                        if pluginapi.get_utils().validatePubKey(distributor):
+                        if stringvalidators.validate_pub_key(distributor):
                             pluginslist[pluginname] = distributor
 
                     logger.debug('Found %s records in repository.' % len(pluginslist), terminal=True)

onionr/static-data/default-plugins/pms/main.py

@@ -23,6 +23,7 @@ import logger, config, threading, time, datetime
 from onionrblockapi import Block
 import onionrexceptions
 from onionrusers import onionrusers
+from onionrutils import stringvalidators
 import locale, sys, os, json
 
 locale.setlocale(locale.LC_ALL, '')
@@ -217,7 +218,7 @@ class OnionrMail:
                     recip = logger.readline('Enter peer address, or -q to stop:').strip()
                     if recip in ('-q', 'q'):
                         raise EOFError
-                    if not self.myCore._utils.validatePubKey(recip):
+                    if not stringvalidators.validate_pub_key(recip):
                         raise onionrexceptions.InvalidPubkey('Must be a valid ed25519 base32 encoded public key')
                 except onionrexceptions.InvalidPubkey:
                     logger.warn('Invalid public key', terminal=True)

onionr/subprocesspow.py

@@ -23,6 +23,7 @@ import subprocess, os
 import multiprocessing, threading, time, json
 from multiprocessing import Pipe, Process
 import core, onionrblockapi, config, onionrutils, logger, onionrproofs
+from onionrutils import bytesconverter
 
 class SubprocessPOW:
     def __init__(self, data, metadata, core_inst=None, subproc_count=None):
@@ -51,7 +52,7 @@ class SubprocessPOW:
         # dump dict to measure bytes of json metadata. Cannot reuse later because the pow token must be added
         json_metadata = json.dumps(metadata).encode()
 
-        self.data = onionrutils.OnionrUtils.strToBytes(data)
+        self.data = bytesconverter.str_to_bytes(data)
         # Calculate difficulty. Dumb for now, may use good algorithm in the future.
         self.difficulty = onionrproofs.getDifficultyForNewBlock(bytes(json_metadata + b'\n' + self.data), coreInst=self.core_inst)
         
@@ -111,7 +112,7 @@ class SubprocessPOW:
             payload = json.dumps(metadata).encode() + b'\n' + data
             # Check sha3_256 hash of block, compare to puzzle. Send payload if puzzle finished
             token = mcore._crypto.sha3Hash(payload)
-            token = onionrutils.OnionrUtils.bytesToStr(token) # ensure token is string
+            token = bytesconverter.bytes_to_str(token) # ensure token is string
             if puzzle == token[0:difficulty]:
                 pipe.send(payload)
                 break

onionr/tests/test_highlevelcrypto.py

@@ -4,6 +4,7 @@ sys.path.append(".")
 import unittest, uuid, hashlib, base64
 import nacl.exceptions
 import nacl.signing, nacl.hash, nacl.encoding
+from onionrutils import stringvalidators, mnemonickeys
 TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
 print("Test directory:", TEST_DIR)
 os.environ["ONIONR_HOME"] = TEST_DIR
@@ -45,19 +46,12 @@ class OnionrCryptoTests(unittest.TestCase):
         self.assertEqual(crypto.sha3Hash(b'test'), normal)
         
     def valid_default_id(self):
-        self.assertTrue(c._utils.validatePubKey(crypto.pubKey))
+        self.assertTrue(stringvalidators.validate_pub_key(crypto.pubKey))
     
     def test_human_readable_length(self):
-        human = c._utils.getHumanReadableID()
+        human = mnemonickeys.get_human_readable_ID(c)
         self.assertTrue(len(human.split(' ')) == 32)
     
-    def test_human_readable_rebuild(self):
-        return # Broken right now
-        # Test if we can get the human readable id, and convert it back to valid base32 key
-        human = c._utils.getHumanReadableID()
-        unHuman = c._utils.convertHumanReadableID(human)
-        nacl.signing.VerifyKey(c._utils.convertHumanReadableID(human), encoder=nacl.encoding.Base32Encoder)
-    
     def test_safe_compare(self):
         self.assertTrue(crypto.safeCompare('test', 'test'))
         self.assertTrue(crypto.safeCompare('test', b'test'))
@@ -130,7 +124,7 @@ class OnionrCryptoTests(unittest.TestCase):
     def test_deterministic(self):
         password = os.urandom(32)
         gen = crypto.generateDeterministic(password)
-        self.assertTrue(c._utils.validatePubKey(gen[0]))
+        self.assertTrue(stringvalidators.validate_pub_key(gen[0]))
         try:
             crypto.generateDeterministic('weakpassword')
         except onionrexceptions.PasswordStrengthError:
@@ -151,6 +145,6 @@ class OnionrCryptoTests(unittest.TestCase):
         gen2 = crypto.generateDeterministic(password)
         self.assertFalse(gen == gen1)
         self.assertTrue(gen1 == gen2)
-        self.assertTrue(c._utils.validatePubKey(gen1[0]))
+        self.assertTrue(stringvalidators.validate_pub_key(gen1[0]))
 
 unittest.main()