From fe4261c4a2d8f5eb53df7b5a87f0342b900371e2 Mon Sep 17 00:00:00 2001 From: Arinerron Date: Sat, 12 May 2018 20:45:32 -0700 Subject: [PATCH] Various improvements - Adds a lot more to the pluginmanager - Refactors code - Relocates functions --- onionr/communicator.py | 75 +++----------- onionr/core.py | 17 ++-- onionr/onionr.py | 2 +- onionr/onionrcrypto.py | 45 +++++++-- onionr/onionrpluginapi.py | 23 +++-- onionr/onionrutils.py | 27 +++--- .../default-plugins/pluginmanager/main.py | 97 ++++++++++++++++--- 7 files changed, 172 insertions(+), 114 deletions(-) diff --git a/onionr/communicator.py b/onionr/communicator.py index 34f4edd3..74355c0e 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -565,31 +565,26 @@ class OnionrCommunicate: except AttributeError: pass - if not self.verifyPow(blockContent, blockMeta2): + if not self._crypto.verifyPow(blockContent, blockMeta2): logger.warn("%s has invalid or insufficient proof of work token, deleting..." % str(i)) self._core.removeBlock(i) continue - - try: - blockMetadata['sig'] - blockMeta2['id'] - except KeyError: - pass else: - #blockData = json.dumps(blockMetadata['meta']) + blockMetadata[blockMetadata.rfind(b'}') + 1:] + if (('sig' in blockMetadata) and ('id' in blockMeta2)): + #blockData = json.dumps(blockMetadata['meta']) + blockMetadata[blockMetadata.rfind(b'}') + 1:] - creator = self._utils.getPeerByHashId(blockMeta2['id']) - try: - creator = creator.decode() - except AttributeError: - pass + creator = self._utils.getPeerByHashId(blockMeta2['id']) + try: + creator = creator.decode() + except AttributeError: + pass - if self._core._crypto.edVerify(blockMetaData['meta'] + blockContent, creator, blockMetadata['sig'], encodedData=True): - logger.info('%s was signed' % str(i)) - self._core.updateBlockInfo(i, 'sig', 'true') - else: - logger.warn('%s has an invalid signature' % str(i)) - self._core.updateBlockInfo(i, 'sig', 'false') + if self._core._crypto.edVerify(blockMetaData['meta'] + blockContent, creator, blockMetadata['sig'], encodedData=True): + logger.info('%s was signed' % str(i)) + self._core.updateBlockInfo(i, 'sig', 'true') + else: + logger.warn('%s has an invalid signature' % str(i)) + self._core.updateBlockInfo(i, 'sig', 'false') try: logger.info('Block type is %s' % str(blockMeta2['type'])) self._core.updateBlockInfo(i, 'dataType', blockMeta2['type']) @@ -605,12 +600,7 @@ class OnionrCommunicate: return def removeBlockFromProcessingList(self, block): - try: - self.blocksProcessing.remove(block) - except ValueError: - return False - else: - return True + return block in blocksProcessing def downloadBlock(self, hash, peerTries=3): ''' @@ -666,41 +656,6 @@ class OnionrCommunicate: return retVal - def verifyPow(self, blockContent, metadata): - ''' - Verifies the proof of work associated with a block - ''' - retData = False - try: - metadata['powToken'] - metadata['powHash'] - token = metadata['powToken'] - except KeyError: - return False - dataLen = len(blockContent) - - expectedHash = self._crypto.blake2bHash(base64.b64decode(metadata['powToken']) + self._crypto.blake2bHash(blockContent.encode())) - difficulty = 0 - try: - expectedHash = expectedHash.decode() - except AttributeError: - pass - if metadata['powHash'] == expectedHash: - difficulty = math.floor(dataLen/1000000) - - mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode() - puzzle = mainHash[0:difficulty] - - if metadata['powHash'][0:difficulty] == puzzle: - logger.info('Validated block pow') - retData = True - else: - logger.warn("Invalid token (#1)") - else: - logger.warn('Invalid token (#2): Expected hash %s, got hash %s...' % (metadata['powHash'], expectedHash)) - - return retData - def urlencode(self, data): ''' URL encodes the data diff --git a/onionr/core.py b/onionr/core.py index 27cd3a37..534a2c03 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -277,11 +277,12 @@ class Core: return - def getData(self,hash): + def getData(self, hash): ''' Simply return the data associated to a hash ''' try: + # logger.debug('Opening %s' % (str(self.blockDataLocation) + str(hash) + '.dat')) dataFile = open(self.blockDataLocation + hash + '.dat', 'rb') data = dataFile.read() dataFile.close() @@ -576,22 +577,22 @@ class Core: return - def getBlockList(self, unsaved = False): + def getBlockList(self, unsaved = False): # TODO: Use unsaved ''' Get list of our blocks ''' conn = sqlite3.connect(self.blockDB) c = conn.cursor() - retData = '' if unsaved: execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();' else: execute = 'SELECT hash FROM hashes ORDER BY RANDOM();' + rows = list() for row in c.execute(execute): for i in row: - retData += i + "\n" + rows.append(i) - return retData + return rows def getBlocksByType(self, blockType): ''' @@ -599,14 +600,14 @@ class Core: ''' conn = sqlite3.connect(self.blockDB) c = conn.cursor() - retData = '' execute = 'SELECT hash FROM hashes WHERE dataType=?;' args = (blockType,) + rows = list() for row in c.execute(execute, args): for i in row: - retData += i + "\n" + rows.append(i) - return retData.split('\n') + return rows def setBlockType(self, hash, blockType): ''' diff --git a/onionr/onionr.py b/onionr/onionr.py index 643bb8ef..53683ec9 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -108,7 +108,7 @@ class Onionr: if not os.path.exists('data/blocks/'): os.mkdir('data/blocks/') - # Copy default plugins into plugins folder + # Copy default plugins into plugins folder if not os.path.exists(plugins.get_plugins_folder()): if os.path.exists('static-data/default-plugins/'): names = [f for f in os.listdir("static-data/default-plugins/") if not os.path.isfile(f)] diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index d400e8b6..315a7b2f 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -26,7 +26,7 @@ class OnionrCrypto: self.keyPowFile = 'data/keyPow.txt' self.pubKey = None self.privKey = None - + self.pubKeyPowToken = None self.pubKeyPowHash = None @@ -89,7 +89,7 @@ class OnionrCrypto: except nacl.exceptions.BadSignatureError: pass return retData - + def edSign(self, data, key, encodeResult=False): '''Ed25519 sign data''' try: @@ -199,7 +199,7 @@ class OnionrCrypto: if returnEncoded: decrypted = base64.b64encode(decrypted) return decrypted - + def generateSymmetricPeer(self, peer): '''Generate symmetric key for a peer and save it to the peer database''' key = self.generateSymmetric() @@ -215,7 +215,7 @@ class OnionrCrypto: private_key = nacl.signing.SigningKey.generate() public_key = private_key.verify_key.encode(encoder=nacl.encoding.Base32Encoder()) return (public_key.decode(), private_key.encode(encoder=nacl.encoding.Base32Encoder()).decode()) - + def pubKeyHashID(self, pubkey=''): '''Accept a ed25519 public key, return a truncated result of X many sha3_256 hash rounds''' if pubkey == '': @@ -237,10 +237,43 @@ class OnionrCrypto: hasher = hashlib.sha3_256() hasher.update(data) return hasher.hexdigest() - + def blake2bHash(self, data): try: data = data.encode() except AttributeError: pass - return nacl.hash.blake2b(data) \ No newline at end of file + return nacl.hash.blake2b(data) + + def verifyPow(self, blockContent, metadata): + ''' + Verifies the proof of work associated with a block + ''' + retData = False + + if not (('powToken' in metadata) and ('powHash' in metadata)): + return False + + dataLen = len(blockContent) + + expectedHash = self.blake2bHash(base64.b64decode(metadata['powToken']) + self.blake2bHash(blockContent.encode())) + difficulty = 0 + try: + expectedHash = expectedHash.decode() + except AttributeError: + pass + if metadata['powHash'] == expectedHash: + difficulty = math.floor(dataLen / 1000000) + + mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode() + puzzle = mainHash[:difficulty] + + if metadata['powHash'][:difficulty] == puzzle: + logger.info('Validated block pow') + retData = True + else: + logger.warn("Invalid token (#1)") + else: + logger.warn('Invalid token (#2): Expected hash %s, got hash %s...' % (metadata['powHash'], expectedHash)) + + return retData diff --git a/onionr/onionrpluginapi.py b/onionr/onionrpluginapi.py index 7aa2b891..51fca1a9 100644 --- a/onionr/onionrpluginapi.py +++ b/onionr/onionrpluginapi.py @@ -18,7 +18,7 @@ along with this program. If not, see . ''' -import onionrplugins as plugins, logger +import onionrplugins, logger class DaemonAPI: def __init__(self, pluginapi): @@ -52,34 +52,34 @@ class PluginAPI: self.pluginapi = pluginapi def start(self, name): - plugins.start(name) + onionrplugins.start(name) def stop(self, name): - plugins.stop(name) + onionrplugins.stop(name) def reload(self, name): - plugins.reload(name) + onionrplugins.reload(name) def enable(self, name): - plugins.enable(name) + onionrplugins.enable(name) def disable(self, name): - plugins.disable(name) + onionrplugins.disable(name) def event(self, name, data = {}): events.event(name, data = data, onionr = self.pluginapi.get_onionr()) def is_enabled(self, name): - return plugins.is_enabled(name) + return onionrplugins.is_enabled(name) def get_enabled_plugins(self): - return plugins.get_enabled() + return onionrplugins.get_enabled() def get_folder(self, name = None, absolute = True): - return plugins.get_plugins_folder(name = name, absolute = absolute) + return onionrplugins.get_plugins_folder(name = name, absolute = absolute) def get_data_folder(self, name, absolute = True): - return plugins.get_plugin_data_folder(name, absolute = absolute) + return onionrplugins.get_plugin_data_folder(name, absolute = absolute) def daemon_event(self, event, plugin = None): return # later make local command like /client/?action=makeEvent&event=eventname&module=modulename @@ -153,6 +153,9 @@ class pluginapi: def get_utils(self): return self.get_onionr().onionrUtils + def get_crypto(): + return self.get_core().crypto + def get_daemonapi(self): return self.daemon diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 0270023a..adc10d34 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -71,7 +71,7 @@ class OnionrUtils: if block == '': logger.error('Could not send PM') else: - logger.info('Sent PM, hash: ' + block) + logger.info('Sent PM, hash: %s' % block) except Exception as error: logger.error('Failed to send PM.', error=error) @@ -103,14 +103,14 @@ class OnionrUtils: for key in newKeyList.split(','): key = key.split('-') if len(key[0]) > 60 or len(key[1]) > 1000: - logger.warn(key[0] + ' or its pow value is too large.') + logger.warn('%s or its pow value is too large.' % key[0]) continue if self._core._crypto.blake2bHash(base64.b64decode(key[1]) + key[0].encode()).startswith('0000'): if not key[0] in self._core.listPeers(randomOrder=False) and type(key) != None and key[0] != self._core._crypto.pubKey: if self._core.addPeer(key[0], key[1]): retVal = True else: - logger.warn(key[0] + 'pow failed') + logger.warn('%s pow failed' % key[0]) return retVal except Exception as error: logger.error('Failed to merge keys.', error=error) @@ -127,7 +127,7 @@ class OnionrUtils: for adder in newAdderList.split(','): if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress(): if self._core.addAddress(adder): - logger.info('Added ' + adder + ' to db.', timestamp = True) + logger.info('Added %s to db.' % adder, timestamp = True) retVal = True else: logger.debug('%s is either our address or already in our DB' % adder) @@ -156,7 +156,7 @@ class OnionrUtils: retData = requests.get('http://' + open('data/host.txt', 'r').read() + ':' + str(config.get('client')['port']) + '/client/?action=' + command + '&token=' + str(config.get('client')['client_hmac']) + '&timingToken=' + self.timingToken).text except Exception as error: if not silent: - logger.error('Failed to make local request (command: ' + str(command) + ').', error=error) + logger.error('Failed to make local request (command: %s).' % command, error=error) retData = False return retData @@ -362,8 +362,7 @@ class OnionrUtils: except json.decoder.JSONDecodeError: pass else: - print('--------------------') - logger.info('Decrypted ' + i + ':') + logger.info('Decrypted %s:' % i) logger.info(message["msg"]) signer = message["id"] @@ -371,16 +370,16 @@ class OnionrUtils: if self.validatePubKey(signer): if self._core._crypto.edVerify(message["msg"], signer, sig, encodedData=True): - logger.info("Good signature by " + signer) + logger.info("Good signature by %s" % signer) else: - logger.warn("Bad signature by " + signer) + logger.warn("Bad signature by %s" % signer) else: - logger.warn("Bad sender id: " + signer) + logger.warn('Bad sender id: %s' % signer) except FileNotFoundError: pass except Exception as error: - logger.error('Failed to open block ' + str(i) + '.', error=error) + logger.error('Failed to open block %s.' % i, error=error) return def getPeerByHashId(self, hash): @@ -438,14 +437,14 @@ class OnionrUtils: scanDir += '/' for block in glob.glob(scanDir + "*.dat"): if block.replace(scanDir, '').replace('.dat', '') not in blockList: - logger.info("Found new block on dist " + block) + logger.info('Found new block on dist %s' % block) with open(block, 'rb') as newBlock: block = block.replace(scanDir, '').replace('.dat', '') if self._core._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''): self._core.addToBlockDB(block.replace('.dat', ''), dataSaved=True) - logger.info('Imported block.') + logger.info('Imported block %s.' % block) else: - logger.warn('Failed to verify hash for ' + block) + logger.warn('Failed to verify hash for %s' % block) def progressBar(self, value = 0, endvalue = 100, width = None): diff --git a/onionr/static-data/default-plugins/pluginmanager/main.py b/onionr/static-data/default-plugins/pluginmanager/main.py index d4f3129a..fb330fd6 100644 --- a/onionr/static-data/default-plugins/pluginmanager/main.py +++ b/onionr/static-data/default-plugins/pluginmanager/main.py @@ -4,7 +4,7 @@ # useful libraries import logger, config -import os, sys, json, time, random +import os, sys, json, time, random, shutil, base64, getpass, datetime plugin_name = 'pluginmanager' @@ -56,6 +56,78 @@ def check(): if not os.path.isfile(keys_file): writeKeys() +# plugin management + +def installBlock(hash, overwrite = True): + logger.debug('install') + +def pluginToBlock(plugin, import_block = True): + try: + directory = pluginapi.get_pluginapi().get_folder(plugin) + data_directory = pluginapi.get_pluginapi().get_data_folder(plugin) + zipfile = pluginapi.get_pluginapi().get_data_folder(plugin_name) + 'plugin.zip' + + if os.path.exists(directory) and not os.path.isfile(directory): + if os.path.exists(data_directory) and not os.path.isfile(data_directory): + shutil.rmtree(data_directory) + if os.path.exists(zipfile) and os.path.isfile(zipfile): + os.remove(zipfile) + if os.path.exists(directory + '__pycache__') and not os.path.isfile(directory + '__pycache__'): + shutil.rmtree(directory + '__pycache__') + + shutil.make_archive(zipfile[:-4], 'zip', directory) + data = base64.b64encode(open(zipfile, 'rb').read()) + + metadata = {'author' : getpass.getuser(), 'date' : str(datetime.datetime.now()), 'content' : data.decode('utf-8'), 'name' : plugin, 'compiled-by' : plugin_name} + + hash = pluginapi.get_core().insertBlock(json.dumps(metadata), header = 'plugin', sign = True) + + if import_block: + pluginapi.get_utils().importNewBlocks() + + return hash + else: + logger.error('Plugin %s does not exist.' % plugin) + except Exception as e: + logger.error('Failed to convert plugin to block.', error = e, timestamp = False) + + return False + +def parseBlock(hash, key):# deal with block metadata + blockContent = pluginapi.get_core().getData(hash) + + try: + blockMetadata = json.loads(blockContent[:blockContent.decode().find(b'}') + 1].decode()) + try: + blockMeta2 = json.loads(blockMetadata['meta']) + except KeyError: + blockMeta2 = {'type': ''} + pass + blockContent = blockContent[blockContent.rfind(b'}') + 1:] + try: + blockContent = blockContent.decode() + except AttributeError: + pass + + if not pluginapi.get_crypto().verifyPow(blockContent, blockMeta2): + logger.debug("(pluginmanager): %s has invalid or insufficient proof of work" % str(hash)) + return False + + if not (('sig' in blockMetadata) and ('id' in blockMeta2)): + logger.debug('(pluginmanager): %s is missing required parameters' % hash) + return False + else: + if pluginapi.get_crypto().edVerify(blockMetaData['meta'] + blockContent, key, blockMetadata['sig'], encodedData=True): + logger.debug('(pluginmanager): %s was signed' % str(hash)) + return True + else: + logger.debug('(pluginmanager): %s has an invalid signature' % str(hash)) + return False + except json.decoder.JSONDecodeError as e: + logger.error('(pluginmanager): Could not decode block metadata.', error = e, timestamp = False) + + return False + # command handlers def help(): @@ -101,13 +173,7 @@ def commandInstallPlugin(): blockhash = str(pkobh) logger.debug('Using block %s...' % blockhash) - logger.info('Downloading plugin...') - for i in range(0, 100): - pluginapi.get_utils().progressBar(i, 100) - time.sleep(random.random() / 5) - logger.info('Finished downloading plugin, verifying and installing...') - time.sleep(1) - logger.info('Installation successful.') + installBlock(blockhash) elif valid_key and not real_key: logger.error('Public key not found. Try adding the node by address manually, if possible.') logger.debug('Is valid key, but the key is not a known one.') @@ -117,13 +183,14 @@ def commandInstallPlugin(): saveKey(pluginname, pkobh) - logger.info('Downloading plugin...') - for i in range(0, 100): - pluginapi.get_utils().progressBar(i, 100) - time.sleep(random.random() / 5) - logger.info('Finished downloading plugin, verifying and installing...') - time.sleep(1) - logger.info('Installation successful.') + blocks = pluginapi.get_core().getBlocksByType('plugin') + + for hash in blocks: + logger.debug('Scanning block for plugin: %s' % hash) + if parseBlock(hash, publickey): + logger.info('success') + else: + logger.warn('fail') else: logger.error('Unknown data "%s"; must be public key or block hash.' % str(pkobh)) return