From 50e93f46e40ba95f3535cbfb70f6c20256613c51 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 22 Jun 2019 16:16:12 -0500 Subject: [PATCH] * Major core refactoring * renamed clandestine to esoteric --- onionr/communicatorutils/downloadblocks.py | 2 +- onionr/core.py | 375 ++---------------- onionr/coredb/__init__.py | 1 + onionr/coredb/blockmetadb/__init__.py | 58 +++ onionr/coredb/blockmetadb/expiredblocks.py | 15 + onionr/coredb/blockmetadb/updateblockinfo.py | 13 + onionr/coredb/daemonqueue/__init__.py | 75 ++++ onionr/coredb/keydb/__init__.py | 1 + onionr/coredb/keydb/addkeys.py | 70 ++++ onionr/coredb/keydb/listkeys.py | 63 +++ onionr/coredb/keydb/removekeys.py | 19 + onionr/coredb/keydb/transportinfo.py | 53 +++ onionr/coredb/keydb/userinfo.py | 50 +++ onionr/onionrstorage.py | 4 +- .../{clandestine => esoteric}/controlapi.py | 12 +- .../{clandestine => esoteric}/info.json | 2 +- .../{clandestine => esoteric}/main.py | 10 +- .../{clandestine => esoteric}/peerserver.py | 12 +- .../static-data/default-plugins/flow/main.py | 1 + onionr/static-data/www/private/index.html | 2 +- 20 files changed, 472 insertions(+), 366 deletions(-) create mode 100644 onionr/coredb/__init__.py create mode 100644 onionr/coredb/blockmetadb/__init__.py create mode 100644 onionr/coredb/blockmetadb/expiredblocks.py create mode 100644 onionr/coredb/blockmetadb/updateblockinfo.py create mode 100644 onionr/coredb/daemonqueue/__init__.py create mode 100644 onionr/coredb/keydb/__init__.py create mode 100644 onionr/coredb/keydb/addkeys.py create mode 100644 onionr/coredb/keydb/listkeys.py create mode 100644 onionr/coredb/keydb/removekeys.py create mode 100644 onionr/coredb/keydb/transportinfo.py create mode 100644 onionr/coredb/keydb/userinfo.py rename onionr/static-data/default-plugins/{clandestine => esoteric}/controlapi.py (83%) rename onionr/static-data/default-plugins/{clandestine => esoteric}/info.json (64%) rename onionr/static-data/default-plugins/{clandestine => esoteric}/main.py (94%) rename onionr/static-data/default-plugins/{clandestine => esoteric}/peerserver.py (78%) diff --git a/onionr/communicatorutils/downloadblocks.py b/onionr/communicatorutils/downloadblocks.py index aad5f884..cb72bd12 100755 --- a/onionr/communicatorutils/downloadblocks.py +++ b/onionr/communicatorutils/downloadblocks.py @@ -110,7 +110,7 @@ def download_blocks_from_communicator(comm_inst): if removeFromQueue: try: del comm_inst.blockQueue[blockHash] # remove from block queue both if success or false - logger.info('%s blocks remaining in queue' % [len(comm_inst.blockQueue)]) + logger.info('%s blocks remaining in queue' % [len(comm_inst.blockQueue)], terminal=True) except KeyError: pass comm_inst.currentDownloading.remove(blockHash) diff --git a/onionr/core.py b/onionr/core.py index be309309..735358e5 100755 --- a/onionr/core.py +++ b/onionr/core.py @@ -20,6 +20,7 @@ import sqlite3, os, sys, time, json, uuid import logger, netcontroller, config from onionrblockapi import Block +import coredb import deadsimplekv as simplekv import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions import onionrblacklist @@ -128,89 +129,19 @@ class Core: ''' Adds a public key to the key database (misleading function name) ''' - if peerID in self.listPeers() or peerID == self._crypto.pubKey: - raise ValueError("specified id is already known") - - # This function simply adds a peer to the DB - if not self._utils.validatePubKey(peerID): - return False - - events.event('pubkey_add', data = {'key': peerID}, onionr = self.onionrInst) - - conn = sqlite3.connect(self.peerDB, timeout=30) - hashID = self._crypto.pubKeyHashID(peerID) - c = conn.cursor() - t = (peerID, name, 'unknown', hashID, 0) - - for i in c.execute("SELECT * FROM peers WHERE id = ?;", (peerID,)): - try: - if i[0] == peerID: - conn.close() - return False - except ValueError: - pass - except IndexError: - pass - c.execute('INSERT INTO peers (id, name, dateSeen, hashID, trust) VALUES(?, ?, ?, ?, ?);', t) - conn.commit() - conn.close() - - return True + return coredb.keydb.addkeys.add_peer(self, peerID, name) def addAddress(self, address): ''' Add an address to the address database (only tor currently) ''' - - if type(address) is None or len(address) == 0: - return False - if self._utils.validateID(address): - if address == config.get('i2p.ownAddr', None) or address == self.hsAddress: - return False - conn = sqlite3.connect(self.addressDB, timeout=30) - c = conn.cursor() - # check if address is in database - # this is safe to do because the address is validated above, but we strip some chars here too just in case - address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '') - for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)): - try: - if i[0] == address: - conn.close() - return False - except ValueError: - pass - except IndexError: - pass - - t = (address, 1) - c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t) - conn.commit() - conn.close() - - events.event('address_add', data = {'address': address}, onionr = self.onionrInst) - - return True - else: - #logger.debug('Invalid ID: %s' % address) - return False + return coredb.keydb.addkeys.add_address(self, address) def removeAddress(self, address): ''' Remove an address from the address database ''' - - if self._utils.validateID(address): - conn = sqlite3.connect(self.addressDB, timeout=30) - c = conn.cursor() - t = (address,) - c.execute('Delete from adders where address=?;', t) - conn.commit() - conn.close() - - events.event('address_remove', data = {'address': address}, onionr = self.onionrInst) - return True - else: - return False + return coredb.keydb.removekeys.remove_address(self, address) def removeBlock(self, block): ''' @@ -272,17 +203,6 @@ class Core: conn.commit() conn.close() - return - - def getData(self, hash): - ''' - Simply return the data associated to a hash - ''' - - data = onionrstorage.getData(self, hash) - - return data - def setData(self, data): ''' Set the data assciated with a hash @@ -299,10 +219,9 @@ class Core: if type(dataHash) is bytes: dataHash = dataHash.decode() blockFileName = self.blockDataLocation + dataHash + '.dat' - if os.path.exists(blockFileName): - pass # TODO: properly check if block is already saved elsewhere - #raise Exception("Data is already set for " + dataHash) - else: + try: + onionrstorage.getData(self, dataHash) + except onionrexceptions.NoDataAvailable: if self._utils.storageCounter.addBytes(dataSize) != False: onionrstorage.store(self, data, blockHash=dataHash) conn = sqlite3.connect(self.blockDB, timeout=30) @@ -314,126 +233,48 @@ class Core: nonceFile.write(dataHash + '\n') else: raise onionrexceptions.DiskAllocationReached + else: + raise Exception("Data is already set for " + dataHash) return dataHash + def getData(self, hash): + ''' + Simply return the data associated to a hash + ''' + return onionrstorage.getData(self, hash) + def daemonQueue(self): ''' Gives commands to the communication proccess/daemon by reading an sqlite3 database This function intended to be used by the client. Queue to exchange data between "client" and server. ''' - - retData = False - if not os.path.exists(self.queueDB): - self.dbCreate.createDaemonDB() - else: - conn = sqlite3.connect(self.queueDB, timeout=30) - c = conn.cursor() - try: - for row in c.execute('SELECT command, data, date, min(ID), responseID FROM commands group by id'): - retData = row - break - except sqlite3.OperationalError: - self.dbCreate.createDaemonDB() - else: - if retData != False: - c.execute('DELETE FROM commands WHERE id=?;', (retData[3],)) - conn.commit() - conn.close() - - events.event('queue_pop', data = {'data': retData}, onionr = self.onionrInst) - - return retData + return coredb.daemonqueue.daemon_queue(self) def daemonQueueAdd(self, command, data='', responseID=''): ''' Add a command to the daemon queue, used by the communication daemon (communicator.py) ''' - - retData = True - - date = self._utils.getEpoch() - conn = sqlite3.connect(self.queueDB, timeout=30) - c = conn.cursor() - t = (command, data, date, responseID) - try: - c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t) - conn.commit() - except sqlite3.OperationalError: - retData = False - self.daemonQueue() - events.event('queue_push', data = {'command': command, 'data': data}, onionr = self.onionrInst) - conn.close() - return retData + return coredb.daemonqueue.daemon_queue_add(self, command, data, responseID) def daemonQueueGetResponse(self, responseID=''): ''' Get a response sent by communicator to the API, by requesting to the API ''' - assert len(responseID) > 0 - resp = self._utils.localCommand('queueResponse/' + responseID) - return resp - - def daemonQueueWaitForResponse(self, responseID='', checkFreqSecs=1): - resp = 'failure' - while resp == 'failure': - resp = self.daemonQueueGetResponse(responseID) - time.sleep(1) - return resp - - def daemonQueueSimple(self, command, data='', checkFreqSecs=1): - ''' - A simplified way to use the daemon queue. Will register a command (with optional data) and wait, return the data - Not always useful, but saves time + LOC in some cases. - This is a blocking function, so be careful. - ''' - responseID = str(uuid.uuid4()) # generate unique response ID - self.daemonQueueAdd(command, data=data, responseID=responseID) - return self.daemonQueueWaitForResponse(responseID, checkFreqSecs) + return coredb.daemonqueue.daemon_queue_get_response(responseID) def clearDaemonQueue(self): ''' Clear the daemon queue (somewhat dangerous) ''' - conn = sqlite3.connect(self.queueDB, timeout=30) - c = conn.cursor() - - try: - c.execute('DELETE FROM commands;') - conn.commit() - except: - pass - - conn.close() - events.event('queue_clear', onionr = self.onionrInst) - - return + return coredb.daemonqueue.clear_daemon_queue(self) def listAdders(self, randomOrder=True, i2p=True, recent=0): ''' Return a list of addresses ''' - conn = sqlite3.connect(self.addressDB, timeout=30) - c = conn.cursor() - if randomOrder: - addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();') - else: - addresses = c.execute('SELECT * FROM adders;') - addressList = [] - for i in addresses: - if len(i[0].strip()) == 0: - continue - addressList.append(i[0]) - conn.close() - testList = list(addressList) # create new list to iterate - for address in testList: - try: - if recent > 0 and (self._utils.getEpoch() - self.getAddressInfo(address, 'lastConnect')) > recent: - raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0 - except TypeError: - addressList.remove(address) - return addressList + return coredb.keydb.listkeys.list_adders(self, randomOrder, i2p, recent) def listPeers(self, randomOrder=True, getPow=False, trust=0): ''' @@ -442,35 +283,7 @@ class Core: randomOrder determines if the list should be in a random order trust sets the minimum trust to list ''' - conn = sqlite3.connect(self.peerDB, timeout=30) - c = conn.cursor() - - payload = '' - - if trust not in (0, 1, 2): - logger.error('Tried to select invalid trust.') - return - - if randomOrder: - payload = 'SELECT * FROM peers WHERE trust >= ? ORDER BY RANDOM();' - else: - payload = 'SELECT * FROM peers WHERE trust >= ?;' - - peerList = [] - - for i in c.execute(payload, (trust,)): - try: - if len(i[0]) != 0: - if getPow: - peerList.append(i[0] + '-' + i[1]) - else: - peerList.append(i[0]) - except TypeError: - pass - - conn.close() - - return peerList + return coredb.keydb.listkeys.list_peers(self, randomOrder, getPow, trust) def getPeerInfo(self, peer, info): ''' @@ -483,46 +296,13 @@ class Core: trust int 4 hashID text 5 ''' - conn = sqlite3.connect(self.peerDB, timeout=30) - c = conn.cursor() - - command = (peer,) - infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'trust': 4, 'hashID': 5} - info = infoNumbers[info] - iterCount = 0 - retVal = '' - - for row in c.execute('SELECT * FROM peers WHERE id=?;', command): - for i in row: - if iterCount == info: - retVal = i - break - else: - iterCount += 1 - - conn.close() - - return retVal + return coredb.keydb.userinfo.get_user_info(self, peer, info) def setPeerInfo(self, peer, key, data): ''' Update a peer for a key ''' - - conn = sqlite3.connect(self.peerDB, timeout=30) - c = conn.cursor() - - command = (data, peer) - - # TODO: validate key on whitelist - if key not in ('id', 'name', 'pubkey', 'forwardKey', 'dateSeen', 'trust'): - raise Exception("Got invalid database key when setting peer info") - - c.execute('UPDATE peers SET ' + key + ' = ? WHERE id=?', command) - conn.commit() - conn.close() - - return + return coredb.keydb.userinfo.set_peer_info(self, peer, key, data) def getAddressInfo(self, address, info): ''' @@ -539,117 +319,35 @@ class Core: trust 8 introduced 9 ''' - - conn = sqlite3.connect(self.addressDB, timeout=30) - c = conn.cursor() - - command = (address,) - infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'powValue': 5, 'failure': 6, 'lastConnect': 7, 'trust': 8, 'introduced': 9} - info = infoNumbers[info] - iterCount = 0 - retVal = '' - - for row in c.execute('SELECT * FROM adders WHERE address=?;', command): - for i in row: - if iterCount == info: - retVal = i - break - else: - iterCount += 1 - conn.close() - - return retVal + return coredb.keydb.transportinfo.get_address_info(self, address, info) def setAddressInfo(self, address, key, data): ''' Update an address for a key ''' - - conn = sqlite3.connect(self.addressDB, timeout=30) - c = conn.cursor() - - command = (data, address) - - if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'failure', 'powValue', 'lastConnect', 'lastConnectAttempt', 'trust', 'introduced'): - raise Exception("Got invalid database key when setting address info") - else: - c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command) - conn.commit() - conn.close() - - return + return coredb.keydb.transportinfo.set_address_info(self, address, key, data) def getBlockList(self, dateRec = None, unsaved = False): ''' Get list of our blocks ''' - if dateRec == None: - dateRec = 0 - - conn = sqlite3.connect(self.blockDB, timeout=30) - c = conn.cursor() - - execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;' - args = (dateRec,) - rows = list() - for row in c.execute(execute, args): - for i in row: - rows.append(i) - conn.close() - return rows + return coredb.blockmetadb.get_block_list(self, dateRec, unsaved) def getBlockDate(self, blockHash): ''' Returns the date a block was received ''' - - conn = sqlite3.connect(self.blockDB, timeout=30) - c = conn.cursor() - - execute = 'SELECT dateReceived FROM hashes WHERE hash=?;' - args = (blockHash,) - for row in c.execute(execute, args): - for i in row: - return int(i) - conn.close() - return None + return coredb.blockmetadb.get_block_date(self, blockHash) def getBlocksByType(self, blockType, orderDate=True): ''' Returns a list of blocks by the type ''' - - conn = sqlite3.connect(self.blockDB, timeout=30) - c = conn.cursor() - - if orderDate: - execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;' - else: - execute = 'SELECT hash FROM hashes WHERE dataType=?;' - - args = (blockType,) - rows = list() - - for row in c.execute(execute, args): - for i in row: - rows.append(i) - conn.close() - return rows + return coredb.blockmetadb.get_blocks_by_type(self, blockType, orderDate) def getExpiredBlocks(self): '''Returns a list of expired blocks''' - conn = sqlite3.connect(self.blockDB, timeout=30) - c = conn.cursor() - date = int(self._utils.getEpoch()) - - execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,) - - rows = list() - for row in c.execute(execute): - for i in row: - rows.append(i) - conn.close() - return rows + return coredb.blockmetadb.expiredblocks.get_expired_blocks(self) def updateBlockInfo(self, hash, key, data): ''' @@ -666,18 +364,7 @@ class Core: dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is expire - expire date for a block ''' - - if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'): - return False - - conn = sqlite3.connect(self.blockDB, timeout=30) - c = conn.cursor() - args = (data, hash) - c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args) - conn.commit() - conn.close() - - return True + return coredb.blockmetadb.updateblockinfo def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False): ''' @@ -695,8 +382,6 @@ class Core: createTime = self._utils.getRoundedEpoch() - # check nonce - #print(data) dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data)) try: with open(self.dataNonceFile, 'r') as nonces: diff --git a/onionr/coredb/__init__.py b/onionr/coredb/__init__.py new file mode 100644 index 00000000..39c909ba --- /dev/null +++ b/onionr/coredb/__init__.py @@ -0,0 +1 @@ +from . import keydb, blockmetadb, daemonqueue \ No newline at end of file diff --git a/onionr/coredb/blockmetadb/__init__.py b/onionr/coredb/blockmetadb/__init__.py new file mode 100644 index 00000000..fd408d35 --- /dev/null +++ b/onionr/coredb/blockmetadb/__init__.py @@ -0,0 +1,58 @@ +import sqlite3 +from . import expiredblocks, updateblockinfo +def get_block_list(core_inst, dateRec = None, unsaved = False): + ''' + Get list of our blocks + ''' + if dateRec == None: + dateRec = 0 + + conn = sqlite3.connect(core_inst.blockDB, timeout=30) + c = conn.cursor() + + execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;' + args = (dateRec,) + rows = list() + for row in c.execute(execute, args): + for i in row: + rows.append(i) + conn.close() + return rows + +def get_block_date(core_inst, blockHash): + ''' + Returns the date a block was received + ''' + + conn = sqlite3.connect(core_inst.blockDB, timeout=30) + c = conn.cursor() + + execute = 'SELECT dateReceived FROM hashes WHERE hash=?;' + args = (blockHash,) + for row in c.execute(execute, args): + for i in row: + return int(i) + conn.close() + return None + +def get_blocks_by_type(core_inst, blockType, orderDate=True): + ''' + Returns a list of blocks by the type + ''' + + conn = sqlite3.connect(core_inst.blockDB, timeout=30) + c = conn.cursor() + + if orderDate: + execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;' + else: + execute = 'SELECT hash FROM hashes WHERE dataType=?;' + + args = (blockType,) + rows = list() + + for row in c.execute(execute, args): + for i in row: + rows.append(i) + conn.close() + return rows \ No newline at end of file diff --git a/onionr/coredb/blockmetadb/expiredblocks.py b/onionr/coredb/blockmetadb/expiredblocks.py new file mode 100644 index 00000000..8debbe26 --- /dev/null +++ b/onionr/coredb/blockmetadb/expiredblocks.py @@ -0,0 +1,15 @@ +import sqlite3 +def get_expired_blocks(core_inst): + '''Returns a list of expired blocks''' + conn = sqlite3.connect(core_inst.blockDB, timeout=30) + c = conn.cursor() + date = int(core_inst._utils.getEpoch()) + + execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,) + + rows = list() + for row in c.execute(execute): + for i in row: + rows.append(i) + conn.close() + return rows \ No newline at end of file diff --git a/onionr/coredb/blockmetadb/updateblockinfo.py b/onionr/coredb/blockmetadb/updateblockinfo.py new file mode 100644 index 00000000..05f0fc8a --- /dev/null +++ b/onionr/coredb/blockmetadb/updateblockinfo.py @@ -0,0 +1,13 @@ +import sqlite3 +def update_block_info(core_inst, hash, key, data): + if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'): + return False + + conn = sqlite3.connect(core_inst.blockDB, timeout=30) + c = conn.cursor() + args = (data, hash) + c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args) + conn.commit() + conn.close() + + return True \ No newline at end of file diff --git a/onionr/coredb/daemonqueue/__init__.py b/onionr/coredb/daemonqueue/__init__.py new file mode 100644 index 00000000..8bd21cfd --- /dev/null +++ b/onionr/coredb/daemonqueue/__init__.py @@ -0,0 +1,75 @@ +import sqlite3, os +import onionrevents as events +def daemon_queue(core_inst): + ''' + Gives commands to the communication proccess/daemon by reading an sqlite3 database + + This function intended to be used by the client. Queue to exchange data between "client" and server. + ''' + + retData = False + if not os.path.exists(core_inst.queueDB): + core_inst.dbCreate.createDaemonDB() + else: + conn = sqlite3.connect(core_inst.queueDB, timeout=30) + c = conn.cursor() + try: + for row in c.execute('SELECT command, data, date, min(ID), responseID FROM commands group by id'): + retData = row + break + except sqlite3.OperationalError: + core_inst.dbCreate.createDaemonDB() + else: + if retData != False: + c.execute('DELETE FROM commands WHERE id=?;', (retData[3],)) + conn.commit() + conn.close() + + events.event('queue_pop', data = {'data': retData}, onionr = core_inst.onionrInst) + + return retData + +def daemon_queue_add(core_inst, command, data='', responseID=''): + ''' + Add a command to the daemon queue, used by the communication daemon (communicator.py) + ''' + + retData = True + + date = core_inst._utils.getEpoch() + conn = sqlite3.connect(core_inst.queueDB, timeout=30) + c = conn.cursor() + t = (command, data, date, responseID) + try: + c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t) + conn.commit() + except sqlite3.OperationalError: + retData = False + core_inst.daemonQueue() + events.event('queue_push', data = {'command': command, 'data': data}, onionr = core_inst.onionrInst) + conn.close() + return retData + +def daemon_queue_get_response(core_inst, responseID=''): + ''' + Get a response sent by communicator to the API, by requesting to the API + ''' + assert len(responseID) > 0 + resp = core_inst._utils.localCommand('queueResponse/' + responseID) + return resp + +def clear_daemon_queue(core_inst): + ''' + Clear the daemon queue (somewhat dangerous) + ''' + conn = sqlite3.connect(core_inst.queueDB, timeout=30) + c = conn.cursor() + + try: + c.execute('DELETE FROM commands;') + conn.commit() + except: + pass + + conn.close() + events.event('queue_clear', onionr = core_inst.onionrInst) \ No newline at end of file diff --git a/onionr/coredb/keydb/__init__.py b/onionr/coredb/keydb/__init__.py new file mode 100644 index 00000000..16d602d5 --- /dev/null +++ b/onionr/coredb/keydb/__init__.py @@ -0,0 +1 @@ +from . import addkeys, listkeys, removekeys, userinfo, transportinfo \ No newline at end of file diff --git a/onionr/coredb/keydb/addkeys.py b/onionr/coredb/keydb/addkeys.py new file mode 100644 index 00000000..ac219b1a --- /dev/null +++ b/onionr/coredb/keydb/addkeys.py @@ -0,0 +1,70 @@ +import sqlite3 +import onionrevents as events, config +def add_peer(core_inst, peerID, name=''): + ''' + Adds a public key to the key database (misleading function name) + ''' + if peerID in core_inst.listPeers() or peerID == core_inst._crypto.pubKey: + raise ValueError("specified id is already known") + + # This function simply adds a peer to the DB + if not core_inst._utils.validatePubKey(peerID): + return False + + events.event('pubkey_add', data = {'key': peerID}, onionr = core_inst.onionrInst) + + conn = sqlite3.connect(core_inst.peerDB, timeout=30) + hashID = core_inst._crypto.pubKeyHashID(peerID) + c = conn.cursor() + t = (peerID, name, 'unknown', hashID, 0) + + for i in c.execute("SELECT * FROM peers WHERE id = ?;", (peerID,)): + try: + if i[0] == peerID: + conn.close() + return False + except ValueError: + pass + except IndexError: + pass + c.execute('INSERT INTO peers (id, name, dateSeen, hashID, trust) VALUES(?, ?, ?, ?, ?);', t) + conn.commit() + conn.close() + + return True + +def add_address(core_inst, address): + ''' + Add an address to the address database (only tor currently) + ''' + + if type(address) is None or len(address) == 0: + return False + if core_inst._utils.validateID(address): + if address == config.get('i2p.ownAddr', None) or address == core_inst.hsAddress: + return False + conn = sqlite3.connect(core_inst.addressDB, timeout=30) + c = conn.cursor() + # check if address is in database + # this is safe to do because the address is validated above, but we strip some chars here too just in case + address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '') + for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)): + try: + if i[0] == address: + conn.close() + return False + except ValueError: + pass + except IndexError: + pass + + t = (address, 1) + c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t) + conn.commit() + conn.close() + + events.event('address_add', data = {'address': address}, onionr = core_inst.onionrInst) + + return True + else: + return False diff --git a/onionr/coredb/keydb/listkeys.py b/onionr/coredb/keydb/listkeys.py new file mode 100644 index 00000000..5ab44fb9 --- /dev/null +++ b/onionr/coredb/keydb/listkeys.py @@ -0,0 +1,63 @@ +import sqlite3 +import logger +def list_peers(core_inst, randomOrder=True, getPow=False, trust=0): + ''' + Return a list of public keys (misleading function name) + + randomOrder determines if the list should be in a random order + trust sets the minimum trust to list + ''' + conn = sqlite3.connect(core_inst.peerDB, timeout=30) + c = conn.cursor() + + payload = '' + + if trust not in (0, 1, 2): + logger.error('Tried to select invalid trust.') + return + + if randomOrder: + payload = 'SELECT * FROM peers WHERE trust >= ? ORDER BY RANDOM();' + else: + payload = 'SELECT * FROM peers WHERE trust >= ?;' + + peerList = [] + + for i in c.execute(payload, (trust,)): + try: + if len(i[0]) != 0: + if getPow: + peerList.append(i[0] + '-' + i[1]) + else: + peerList.append(i[0]) + except TypeError: + pass + + conn.close() + + return peerList + +def list_adders(core_inst, randomOrder=True, i2p=True, recent=0): + ''' + Return a list of transport addresses + ''' + conn = sqlite3.connect(core_inst.addressDB, timeout=30) + c = conn.cursor() + if randomOrder: + addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();') + else: + addresses = c.execute('SELECT * FROM adders;') + addressList = [] + for i in addresses: + if len(i[0].strip()) == 0: + continue + addressList.append(i[0]) + conn.close() + testList = list(addressList) # create new list to iterate + for address in testList: + try: + if recent > 0 and (core_inst._utils.getEpoch() - core_inst.getAddressInfo(address, 'lastConnect')) > recent: + raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0 + except TypeError: + addressList.remove(address) + return addressList \ No newline at end of file diff --git a/onionr/coredb/keydb/removekeys.py b/onionr/coredb/keydb/removekeys.py new file mode 100644 index 00000000..10f44a1b --- /dev/null +++ b/onionr/coredb/keydb/removekeys.py @@ -0,0 +1,19 @@ +import sqlite3 +import onionrevents as events +def remove_address(core_inst, address): + ''' + Remove an address from the address database + ''' + + if core_inst._utils.validateID(address): + conn = sqlite3.connect(core_inst.addressDB, timeout=30) + c = conn.cursor() + t = (address,) + c.execute('Delete from adders where address=?;', t) + conn.commit() + conn.close() + + events.event('address_remove', data = {'address': address}, onionr = core_inst.onionrInst) + return True + else: + return False \ No newline at end of file diff --git a/onionr/coredb/keydb/transportinfo.py b/onionr/coredb/keydb/transportinfo.py new file mode 100644 index 00000000..9fd642ec --- /dev/null +++ b/onionr/coredb/keydb/transportinfo.py @@ -0,0 +1,53 @@ +import sqlite3 +def get_address_info(core_inst, address, info): + ''' + Get info about an address from its database entry + + address text, 0 + type int, 1 + knownPeer text, 2 + speed int, 3 + success int, 4 + powValue 5 + failure int 6 + lastConnect 7 + trust 8 + introduced 9 + ''' + + conn = sqlite3.connect(core_inst.addressDB, timeout=30) + c = conn.cursor() + + command = (address,) + infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'powValue': 5, 'failure': 6, 'lastConnect': 7, 'trust': 8, 'introduced': 9} + info = infoNumbers[info] + iterCount = 0 + retVal = '' + + for row in c.execute('SELECT * FROM adders WHERE address=?;', command): + for i in row: + if iterCount == info: + retVal = i + break + else: + iterCount += 1 + conn.close() + + return retVal + +def set_address_info(core_inst, address, key, data): + ''' + Update an address for a key + ''' + + conn = sqlite3.connect(core_inst.addressDB, timeout=30) + c = conn.cursor() + + command = (data, address) + + if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'failure', 'powValue', 'lastConnect', 'lastConnectAttempt', 'trust', 'introduced'): + raise Exception("Got invalid database key when setting address info") + else: + c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command) + conn.commit() + conn.close() \ No newline at end of file diff --git a/onionr/coredb/keydb/userinfo.py b/onionr/coredb/keydb/userinfo.py new file mode 100644 index 00000000..84a737d6 --- /dev/null +++ b/onionr/coredb/keydb/userinfo.py @@ -0,0 +1,50 @@ +import sqlite3 +def get_user_info(core_inst, peer, info): + ''' + Get info about a peer from their database entry + + id text 0 + name text, 1 + adders text, 2 + dateSeen not null, 3 + trust int 4 + hashID text 5 + ''' + conn = sqlite3.connect(core_inst.peerDB, timeout=30) + c = conn.cursor() + + command = (peer,) + infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'trust': 4, 'hashID': 5} + info = infoNumbers[info] + iterCount = 0 + retVal = '' + + for row in c.execute('SELECT * FROM peers WHERE id=?;', command): + for i in row: + if iterCount == info: + retVal = i + break + else: + iterCount += 1 + + conn.close() + + return retVal + +def set_peer_info(core_inst, peer, key, data): + ''' + Update a peer for a key + ''' + + conn = sqlite3.connect(core_inst.peerDB, timeout=30) + c = conn.cursor() + + command = (data, peer) + + # TODO: validate key on whitelist + if key not in ('id', 'name', 'pubkey', 'forwardKey', 'dateSeen', 'trust'): + raise Exception("Got invalid database key when setting peer info") + + c.execute('UPDATE peers SET ' + key + ' = ? WHERE id=?', command) + conn.commit() + conn.close() \ No newline at end of file diff --git a/onionr/onionrstorage.py b/onionr/onionrstorage.py index 5f44c4f3..b859662d 100755 --- a/onionr/onionrstorage.py +++ b/onionr/onionrstorage.py @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import core, sys, sqlite3, os, dbcreator +import core, sys, sqlite3, os, dbcreator, onionrexceptions DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option @@ -94,4 +94,6 @@ def getData(coreInst, bHash): retData = block.read() else: retData = _dbFetch(coreInst, bHash) + if retData is None: + raise onionrexceptions.NoDataAvailable("Block data for %s is not available" % [bHash]) return retData \ No newline at end of file diff --git a/onionr/static-data/default-plugins/clandestine/controlapi.py b/onionr/static-data/default-plugins/esoteric/controlapi.py similarity index 83% rename from onionr/static-data/default-plugins/clandestine/controlapi.py rename to onionr/static-data/default-plugins/esoteric/controlapi.py index e40de9b6..7c298063 100755 --- a/onionr/static-data/default-plugins/clandestine/controlapi.py +++ b/onionr/static-data/default-plugins/esoteric/controlapi.py @@ -22,13 +22,13 @@ from flask import Response, request, redirect, Blueprint, send_from_directory import core core_inst = core.Core() -flask_blueprint = Blueprint('clandestine_control', __name__) +flask_blueprint = Blueprint('esoteric_control', __name__) -@flask_blueprint.route('/clandestine/ping') +@flask_blueprint.route('/esoteric/ping') def ping(): return 'pong!' -@flask_blueprint.route('/clandestine/send/', methods=['POST']) +@flask_blueprint.route('/esoteric/send/', methods=['POST']) def send_message(peer): data = request.get_json(force=True) core_inst.keyStore.refresh() @@ -40,14 +40,14 @@ def send_message(peer): core_inst.keyStore.flush() return Response('success') -@flask_blueprint.route('/clandestine/gets/') +@flask_blueprint.route('/esoteric/gets/') def get_sent(peer): sent = core_inst.keyStore.get('s' + peer) if sent is None: sent = [] return Response(json.dumps(sent)) -@flask_blueprint.route('/clandestine/addrec/', methods=['POST']) +@flask_blueprint.route('/esoteric/addrec/', methods=['POST']) def add_rec(peer): data = request.get_json(force=True) core_inst.keyStore.refresh() @@ -59,7 +59,7 @@ def add_rec(peer): core_inst.keyStore.flush() return Response('success') -@flask_blueprint.route('/clandestine/getrec/') +@flask_blueprint.route('/esoteric/getrec/') def get_messages(peer): core_inst.keyStore.refresh() existing = core_inst.keyStore.get('r' + peer) diff --git a/onionr/static-data/default-plugins/clandestine/info.json b/onionr/static-data/default-plugins/esoteric/info.json similarity index 64% rename from onionr/static-data/default-plugins/clandestine/info.json rename to onionr/static-data/default-plugins/esoteric/info.json index 7c5a143c..6fcc6e0e 100755 --- a/onionr/static-data/default-plugins/clandestine/info.json +++ b/onionr/static-data/default-plugins/esoteric/info.json @@ -1,5 +1,5 @@ { - "name" : "clandestine", + "name" : "esoteric", "version" : "1.0", "author" : "onionr" } diff --git a/onionr/static-data/default-plugins/clandestine/main.py b/onionr/static-data/default-plugins/esoteric/main.py similarity index 94% rename from onionr/static-data/default-plugins/clandestine/main.py rename to onionr/static-data/default-plugins/esoteric/main.py index 29afa3e1..1f4cdd32 100755 --- a/onionr/static-data/default-plugins/clandestine/main.py +++ b/onionr/static-data/default-plugins/esoteric/main.py @@ -24,7 +24,7 @@ locale.setlocale(locale.LC_ALL, '') import onionrservices, logger from onionrservices import bootstrapservice -plugin_name = 'clandestine' +plugin_name = 'esoteric' PLUGIN_VERSION = '0.0.0' sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) import controlapi, peerserver @@ -36,7 +36,7 @@ def exit_with_error(text=''): logger.error(text) sys.exit(1) -class Clandestine: +class Esoteric: def __init__(self, pluginapi): self.myCore = pluginapi.get_core() self.peer = None @@ -58,7 +58,7 @@ class Clandestine: message += '\n' except EOFError: message = json.dumps({'m': message, 't': self.myCore._utils.getEpoch()}) - print(self.myCore._utils.doPostRequest('http://%s/clandestine/sendto' % (self.transport,), port=self.socks, data=message)) + print(self.myCore._utils.doPostRequest('http://%s/esoteric/sendto' % (self.transport,), port=self.socks, data=message)) message = '' except KeyboardInterrupt: self.shutdown = True @@ -89,6 +89,6 @@ def on_init(api, data = None): ''' pluginapi = api - chat = Clandestine(pluginapi) - api.commands.register(['clandestine'], chat.create) + chat = Esoteric(pluginapi) + api.commands.register(['esoteric'], chat.create) return diff --git a/onionr/static-data/default-plugins/clandestine/peerserver.py b/onionr/static-data/default-plugins/esoteric/peerserver.py similarity index 78% rename from onionr/static-data/default-plugins/clandestine/peerserver.py rename to onionr/static-data/default-plugins/esoteric/peerserver.py index 5e4fba25..b3ae423e 100755 --- a/onionr/static-data/default-plugins/clandestine/peerserver.py +++ b/onionr/static-data/default-plugins/esoteric/peerserver.py @@ -21,7 +21,7 @@ import sys, os, json import core from flask import Response, request, redirect, Blueprint, abort, g sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) -direct_blueprint = Blueprint('clandestine', __name__) +direct_blueprint = Blueprint('esoteric', __name__) core_inst = core.Core() storage_dir = core_inst.dataDir @@ -35,11 +35,11 @@ def request_setup(): g.host = host g.peer = core_inst.keyStore.get('dc-' + g.host) -@direct_blueprint.route('/clandestine/ping') +@direct_blueprint.route('/esoteric/ping') def pingdirect(): return 'pong!' -@direct_blueprint.route('/clandestine/sendto', methods=['POST', 'GET']) +@direct_blueprint.route('/esoteric/sendto', methods=['POST', 'GET']) def sendto(): try: msg = request.get_json(force=True) @@ -47,9 +47,9 @@ def sendto(): msg = '' else: msg = json.dumps(msg) - core_inst._utils.localCommand('/clandestine/addrec/%s' % (g.peer,), post=True, postData=msg) + core_inst._utils.localCommand('/esoteric/addrec/%s' % (g.peer,), post=True, postData=msg) return Response('success') -@direct_blueprint.route('/clandestine/poll') +@direct_blueprint.route('/esoteric/poll') def poll_chat(): - return Response(core_inst._utils.localCommand('/clandestine/gets/%s' % (g.peer,))) \ No newline at end of file + return Response(core_inst._utils.localCommand('/esoteric/gets/%s' % (g.peer,))) \ No newline at end of file diff --git a/onionr/static-data/default-plugins/flow/main.py b/onionr/static-data/default-plugins/flow/main.py index 63c86cbc..d1c2c87d 100755 --- a/onionr/static-data/default-plugins/flow/main.py +++ b/onionr/static-data/default-plugins/flow/main.py @@ -61,6 +61,7 @@ class OnionrFlow: self.flowRunning = False expireTime = self.myCore._utils.getEpoch() + 43200 if len(message) > 0: + logger.info('Inserting message as block...', terminal=True) self.myCore.insertBlock(message, header='txt', expire=expireTime, meta={'ch': self.channel}) logger.info("Flow is exiting, goodbye", terminal=True) diff --git a/onionr/static-data/www/private/index.html b/onionr/static-data/www/private/index.html index 9a548ebd..a1650883 100755 --- a/onionr/static-data/www/private/index.html +++ b/onionr/static-data/www/private/index.html @@ -26,7 +26,7 @@


Mail - Friend Manager - Circle - - Clandestine + Esoteric

Edit Configuration