* Major core refactoring

* renamed clandestine to esoteric

onionr/communicatorutils/downloadblocks.py

@@ -110,7 +110,7 @@ def download_blocks_from_communicator(comm_inst):
             if removeFromQueue:
                 try:
                     del comm_inst.blockQueue[blockHash] # remove from block queue both if success or false
-                    logger.info('%s blocks remaining in queue' % [len(comm_inst.blockQueue)])
+                    logger.info('%s blocks remaining in queue' % [len(comm_inst.blockQueue)], terminal=True)
                 except KeyError:
                     pass
         comm_inst.currentDownloading.remove(blockHash)

onionr/core.py

@@ -20,6 +20,7 @@
 import sqlite3, os, sys, time, json, uuid
 import logger, netcontroller, config
 from onionrblockapi import Block
+import coredb
 import deadsimplekv as simplekv
 import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions
 import onionrblacklist
@@ -128,89 +129,19 @@ class Core:
         '''
             Adds a public key to the key database (misleading function name)
         '''
-        if peerID in self.listPeers() or peerID == self._crypto.pubKey:
+        return coredb.keydb.addkeys.add_peer(self, peerID, name)
-            raise ValueError("specified id is already known")
-
-        # This function simply adds a peer to the DB
-        if not self._utils.validatePubKey(peerID):
-            return False
-
-        events.event('pubkey_add', data = {'key': peerID}, onionr = self.onionrInst)
-
-        conn = sqlite3.connect(self.peerDB, timeout=30)
-        hashID = self._crypto.pubKeyHashID(peerID)
-        c = conn.cursor()
-        t = (peerID, name, 'unknown', hashID, 0)
-
-        for i in c.execute("SELECT * FROM peers WHERE id = ?;", (peerID,)):
-            try:
-                if i[0] == peerID:
-                    conn.close()
-                    return False
-            except ValueError:
-                pass
-            except IndexError:
-                pass
-        c.execute('INSERT INTO peers (id, name, dateSeen, hashID, trust) VALUES(?, ?, ?, ?, ?);', t)
-        conn.commit()
-        conn.close()
-
-        return True
 
     def addAddress(self, address):
         '''
             Add an address to the address database (only tor currently)
         '''
-
+        return coredb.keydb.addkeys.add_address(self, address)
-        if type(address) is None or len(address) == 0:
-            return False
-        if self._utils.validateID(address):
-            if address == config.get('i2p.ownAddr', None) or address == self.hsAddress:
-                return False
-            conn = sqlite3.connect(self.addressDB, timeout=30)
-            c = conn.cursor()
-            # check if address is in database
-            # this is safe to do because the address is validated above, but we strip some chars here too just in case
-            address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '')
-            for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)):
-                try:
-                    if i[0] == address:
-                        conn.close()
-                        return False
-                except ValueError:
-                    pass
-                except IndexError:
-                    pass
-
-            t = (address, 1)
-            c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t)
-            conn.commit()
-            conn.close()
-
-            events.event('address_add', data = {'address': address}, onionr = self.onionrInst)
-
-            return True
-        else:
-            #logger.debug('Invalid ID: %s' % address)
-            return False
 
     def removeAddress(self, address):
         '''
             Remove an address from the address database
         '''
-
+        return coredb.keydb.removekeys.remove_address(self, address)
-        if self._utils.validateID(address):
-            conn = sqlite3.connect(self.addressDB, timeout=30)
-            c = conn.cursor()
-            t = (address,)
-            c.execute('Delete from adders where address=?;', t)
-            conn.commit()
-            conn.close()
-
-            events.event('address_remove', data = {'address': address}, onionr = self.onionrInst)
-            return True
-        else:
-            return False
 
     def removeBlock(self, block):
         '''
@@ -272,17 +203,6 @@ class Core:
         conn.commit()
         conn.close()
 
-        return
-
-    def getData(self, hash):
-        '''
-            Simply return the data associated to a hash
-        '''
-
-        data = onionrstorage.getData(self, hash)
-
-        return data
-
     def setData(self, data):
         '''
             Set the data assciated with a hash
@@ -299,10 +219,9 @@ class Core:
         if type(dataHash) is bytes:
             dataHash = dataHash.decode()
         blockFileName = self.blockDataLocation + dataHash + '.dat'
-        if os.path.exists(blockFileName):
+        try:
-            pass # TODO: properly check if block is already saved elsewhere
+            onionrstorage.getData(self, dataHash)
-            #raise Exception("Data is already set for " + dataHash)
+        except onionrexceptions.NoDataAvailable:
-        else:
             if self._utils.storageCounter.addBytes(dataSize) != False:
                 onionrstorage.store(self, data, blockHash=dataHash)
                 conn = sqlite3.connect(self.blockDB, timeout=30)
@@ -314,126 +233,48 @@ class Core:
                     nonceFile.write(dataHash + '\n')
             else:
                 raise onionrexceptions.DiskAllocationReached
+        else:
+            raise Exception("Data is already set for " + dataHash)
 
         return dataHash
 
+    def getData(self, hash):
+        '''
+            Simply return the data associated to a hash
+        '''
+        return onionrstorage.getData(self, hash)
+
     def daemonQueue(self):
         '''
             Gives commands to the communication proccess/daemon by reading an sqlite3 database
 
             This function intended to be used by the client. Queue to exchange data between "client" and server.
         '''
-
+        return coredb.daemonqueue.daemon_queue(self)
-        retData = False
-        if not os.path.exists(self.queueDB):
-            self.dbCreate.createDaemonDB()
-        else:
-            conn = sqlite3.connect(self.queueDB, timeout=30)
-            c = conn.cursor()
-            try:
-                for row in c.execute('SELECT command, data, date, min(ID), responseID FROM commands group by id'):
-                    retData = row
-                    break
-            except sqlite3.OperationalError:
-                self.dbCreate.createDaemonDB()
-            else:
-                if retData != False:
-                    c.execute('DELETE FROM commands WHERE id=?;', (retData[3],))
-            conn.commit()
-            conn.close()
-
-        events.event('queue_pop', data = {'data': retData}, onionr = self.onionrInst)
-
-        return retData
 
     def daemonQueueAdd(self, command, data='', responseID=''):
         '''
             Add a command to the daemon queue, used by the communication daemon (communicator.py)
         '''
-
+        return coredb.daemonqueue.daemon_queue_add(self, command, data, responseID)
-        retData = True
-
-        date = self._utils.getEpoch()
-        conn = sqlite3.connect(self.queueDB, timeout=30)
-        c = conn.cursor()
-        t = (command, data, date, responseID)
-        try:
-            c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t)
-            conn.commit()
-        except sqlite3.OperationalError:
-            retData = False
-            self.daemonQueue()
-        events.event('queue_push', data = {'command': command, 'data': data}, onionr = self.onionrInst)
-        conn.close()
-        return retData
 
     def daemonQueueGetResponse(self, responseID=''):
         '''
             Get a response sent by communicator to the API, by requesting to the API
         '''
-        assert len(responseID) > 0
+        return coredb.daemonqueue.daemon_queue_get_response(responseID)
-        resp = self._utils.localCommand('queueResponse/' + responseID)
-        return resp
-
-    def daemonQueueWaitForResponse(self, responseID='', checkFreqSecs=1):
-        resp = 'failure'
-        while resp == 'failure':
-            resp = self.daemonQueueGetResponse(responseID)
-            time.sleep(1)
-        return resp
-
-    def daemonQueueSimple(self, command, data='', checkFreqSecs=1):
-        '''
-        A simplified way to use the daemon queue. Will register a command (with optional data) and wait, return the data
-        Not always useful, but saves time + LOC in some cases.
-        This is a blocking function, so be careful.
-        '''
-        responseID = str(uuid.uuid4()) # generate unique response ID
-        self.daemonQueueAdd(command, data=data, responseID=responseID)
-        return self.daemonQueueWaitForResponse(responseID, checkFreqSecs)
 
     def clearDaemonQueue(self):
         '''
             Clear the daemon queue (somewhat dangerous)
         '''
-        conn = sqlite3.connect(self.queueDB, timeout=30)
+        return coredb.daemonqueue.clear_daemon_queue(self)
-        c = conn.cursor()
-
-        try:
-            c.execute('DELETE FROM commands;')
-            conn.commit()
-        except:
-            pass
-
-        conn.close()
-        events.event('queue_clear', onionr = self.onionrInst)
-
-        return
 
     def listAdders(self, randomOrder=True, i2p=True, recent=0):
         '''
             Return a list of addresses
         '''
-        conn = sqlite3.connect(self.addressDB, timeout=30)
+        return coredb.keydb.listkeys.list_adders(self, randomOrder, i2p, recent)
-        c = conn.cursor()
-        if randomOrder:
-            addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();')
-        else:
-            addresses = c.execute('SELECT * FROM adders;')
-        addressList = []
-        for i in addresses:
-            if len(i[0].strip()) == 0:
-                continue
-            addressList.append(i[0])
-        conn.close()
-        testList = list(addressList) # create new list to iterate
-        for address in testList:
-            try:
-                if recent > 0 and (self._utils.getEpoch() - self.getAddressInfo(address, 'lastConnect')) > recent:
-                    raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0
-            except TypeError:
-                addressList.remove(address)
-        return addressList
 
     def listPeers(self, randomOrder=True, getPow=False, trust=0):
         '''
@@ -442,35 +283,7 @@ class Core:
             randomOrder determines if the list should be in a random order
             trust sets the minimum trust to list
         '''
-        conn = sqlite3.connect(self.peerDB, timeout=30)
+        return coredb.keydb.listkeys.list_peers(self, randomOrder, getPow, trust)
-        c = conn.cursor()
-
-        payload = ''
-
-        if trust not in (0, 1, 2):
-            logger.error('Tried to select invalid trust.')
-            return
-
-        if randomOrder:
-            payload = 'SELECT * FROM peers WHERE trust >= ? ORDER BY RANDOM();'
-        else:
-            payload = 'SELECT * FROM peers WHERE trust >= ?;'
-
-        peerList = []
-
-        for i in c.execute(payload, (trust,)):
-            try:
-                if len(i[0]) != 0:
-                    if getPow:
-                        peerList.append(i[0] + '-' + i[1])
-                    else:
-                        peerList.append(i[0])
-            except TypeError:
-                pass
-
-        conn.close()
-
-        return peerList
 
     def getPeerInfo(self, peer, info):
         '''
@@ -483,46 +296,13 @@ class Core:
             trust int           4
             hashID text         5
         '''
-        conn = sqlite3.connect(self.peerDB, timeout=30)
+        return coredb.keydb.userinfo.get_user_info(self, peer, info)
-        c = conn.cursor()
-
-        command = (peer,)
-        infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'trust': 4, 'hashID': 5}
-        info = infoNumbers[info]
-        iterCount = 0
-        retVal = ''
-
-        for row in c.execute('SELECT * FROM peers WHERE id=?;', command):
-            for i in row:
-                if iterCount == info:
-                    retVal = i
-                    break
-                else:
-                    iterCount += 1
-
-        conn.close()
-
-        return retVal
 
     def setPeerInfo(self, peer, key, data):
         '''
             Update a peer for a key
         '''
-
+        return coredb.keydb.userinfo.set_peer_info(self, peer, key, data)
-        conn = sqlite3.connect(self.peerDB, timeout=30)
-        c = conn.cursor()
-
-        command = (data, peer)
-
-        # TODO: validate key on whitelist
-        if key not in ('id', 'name', 'pubkey', 'forwardKey', 'dateSeen', 'trust'):
-            raise Exception("Got invalid database key when setting peer info")
-
-        c.execute('UPDATE peers SET ' + key + ' = ? WHERE id=?', command)
-        conn.commit()
-        conn.close()
-
-        return
 
     def getAddressInfo(self, address, info):
         '''
@@ -539,117 +319,35 @@ class Core:
             trust       8
             introduced  9
         '''
-
+        return coredb.keydb.transportinfo.get_address_info(self, address, info)
-        conn = sqlite3.connect(self.addressDB, timeout=30)
-        c = conn.cursor()
-
-        command = (address,)
-        infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'powValue': 5, 'failure': 6, 'lastConnect': 7, 'trust': 8, 'introduced': 9}
-        info = infoNumbers[info]
-        iterCount = 0
-        retVal = ''
-
-        for row in c.execute('SELECT * FROM adders WHERE address=?;', command):
-            for i in row:
-                if iterCount == info:
-                    retVal = i
-                    break
-                else:
-                    iterCount += 1
-        conn.close()
-
-        return retVal
 
     def setAddressInfo(self, address, key, data):
         '''
             Update an address for a key
         '''
-
+        return coredb.keydb.transportinfo.set_address_info(self, address, key, data)
-        conn = sqlite3.connect(self.addressDB, timeout=30)
-        c = conn.cursor()
-
-        command = (data, address)
-
-        if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'failure', 'powValue', 'lastConnect', 'lastConnectAttempt', 'trust', 'introduced'):
-            raise Exception("Got invalid database key when setting address info")
-        else:
-            c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command)
-            conn.commit()
-        conn.close()
-
-        return
 
     def getBlockList(self, dateRec = None, unsaved = False):
         '''
             Get list of our blocks
         '''
-        if dateRec == None:
+        return coredb.blockmetadb.get_block_list(self, dateRec, unsaved)
-            dateRec = 0
-
-        conn = sqlite3.connect(self.blockDB, timeout=30)
-        c = conn.cursor()
-
-        execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
-        args = (dateRec,)
-        rows = list()
-        for row in c.execute(execute, args):
-            for i in row:
-                rows.append(i)
-        conn.close()
-        return rows
 
     def getBlockDate(self, blockHash):
         '''
             Returns the date a block was received
         '''
-
+        return coredb.blockmetadb.get_block_date(self, blockHash)
-        conn = sqlite3.connect(self.blockDB, timeout=30)
-        c = conn.cursor()
-
-        execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
-        args = (blockHash,)
-        for row in c.execute(execute, args):
-            for i in row:
-                return int(i)
-        conn.close()
-        return None
 
     def getBlocksByType(self, blockType, orderDate=True):
         '''
             Returns a list of blocks by the type
         '''
-
+        return coredb.blockmetadb.get_blocks_by_type(self, blockType, orderDate)
-        conn = sqlite3.connect(self.blockDB, timeout=30)
-        c = conn.cursor()
-
-        if orderDate:
-            execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;'
-        else:
-            execute = 'SELECT hash FROM hashes WHERE dataType=?;'
-
-        args = (blockType,)
-        rows = list()
-
-        for row in c.execute(execute, args):
-            for i in row:
-                rows.append(i)
-        conn.close()
-        return rows
 
     def getExpiredBlocks(self):
         '''Returns a list of expired blocks'''
-        conn = sqlite3.connect(self.blockDB, timeout=30)
+        return coredb.blockmetadb.expiredblocks.get_expired_blocks(self)
-        c = conn.cursor()
-        date = int(self._utils.getEpoch())
-
-        execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,)
-
-        rows = list()
-        for row in c.execute(execute):
-            for i in row:
-                rows.append(i)
-        conn.close()
-        return rows
 
     def updateBlockInfo(self, hash, key, data):
         '''
@@ -666,18 +364,7 @@ class Core:
             dateClaimed  - timestamp claimed inside the block, only as trustworthy as the block author is
             expire       - expire date for a block
         '''
-
+        return coredb.blockmetadb.updateblockinfo
-        if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'):
-            return False
-
-        conn = sqlite3.connect(self.blockDB, timeout=30)
-        c = conn.cursor()
-        args = (data, hash)
-        c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args)
-        conn.commit()
-        conn.close()
-
-        return True
 
     def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False):
         '''
@@ -695,8 +382,6 @@ class Core:
 
         createTime = self._utils.getRoundedEpoch()
 
-        # check nonce
-        #print(data)
         dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data))
         try:
             with open(self.dataNonceFile, 'r') as nonces:

onionr/coredb/__init__.py

@@ -0,0 +1 @@
+from . import keydb, blockmetadb, daemonqueue

onionr/coredb/blockmetadb/__init__.py

@@ -0,0 +1,58 @@
+import sqlite3
+from . import expiredblocks, updateblockinfo
+def get_block_list(core_inst, dateRec = None, unsaved = False):
+    '''
+        Get list of our blocks
+    '''
+    if dateRec == None:
+        dateRec = 0
+
+    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
+    c = conn.cursor()
+
+    execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
+    args = (dateRec,)
+    rows = list()
+    for row in c.execute(execute, args):
+        for i in row:
+            rows.append(i)
+    conn.close()
+    return rows
+
+def get_block_date(core_inst, blockHash):
+    '''
+        Returns the date a block was received
+    '''
+
+    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
+    c = conn.cursor()
+
+    execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
+    args = (blockHash,)
+    for row in c.execute(execute, args):
+        for i in row:
+            return int(i)
+    conn.close()
+    return None
+
+def get_blocks_by_type(core_inst, blockType, orderDate=True):
+    '''
+        Returns a list of blocks by the type
+    '''
+
+    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
+    c = conn.cursor()
+
+    if orderDate:
+        execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;'
+    else:
+        execute = 'SELECT hash FROM hashes WHERE dataType=?;'
+
+    args = (blockType,)
+    rows = list()
+
+    for row in c.execute(execute, args):
+        for i in row:
+            rows.append(i)
+    conn.close()
+    return rows

onionr/coredb/blockmetadb/expiredblocks.py

@@ -0,0 +1,15 @@
+import sqlite3
+def get_expired_blocks(core_inst):
+    '''Returns a list of expired blocks'''
+    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
+    c = conn.cursor()
+    date = int(core_inst._utils.getEpoch())
+
+    execute = 'SELECT hash FROM hashes WHERE expire <= %s ORDER BY dateReceived;' % (date,)
+
+    rows = list()
+    for row in c.execute(execute):
+        for i in row:
+            rows.append(i)
+    conn.close()
+    return rows

onionr/coredb/blockmetadb/updateblockinfo.py

@@ -0,0 +1,13 @@
+import sqlite3
+def update_block_info(core_inst, hash, key, data):
+    if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'):
+        return False
+
+    conn = sqlite3.connect(core_inst.blockDB, timeout=30)
+    c = conn.cursor()
+    args = (data, hash)
+    c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args)
+    conn.commit()
+    conn.close()
+
+    return True

onionr/coredb/daemonqueue/__init__.py

@@ -0,0 +1,75 @@
+import sqlite3, os
+import onionrevents as events
+def daemon_queue(core_inst):
+    '''
+        Gives commands to the communication proccess/daemon by reading an sqlite3 database
+
+        This function intended to be used by the client. Queue to exchange data between "client" and server.
+    '''
+
+    retData = False
+    if not os.path.exists(core_inst.queueDB):
+        core_inst.dbCreate.createDaemonDB()
+    else:
+        conn = sqlite3.connect(core_inst.queueDB, timeout=30)
+        c = conn.cursor()
+        try:
+            for row in c.execute('SELECT command, data, date, min(ID), responseID FROM commands group by id'):
+                retData = row
+                break
+        except sqlite3.OperationalError:
+            core_inst.dbCreate.createDaemonDB()
+        else:
+            if retData != False:
+                c.execute('DELETE FROM commands WHERE id=?;', (retData[3],))
+        conn.commit()
+        conn.close()
+
+    events.event('queue_pop', data = {'data': retData}, onionr = core_inst.onionrInst)
+
+    return retData
+
+def daemon_queue_add(core_inst, command, data='', responseID=''):
+    '''
+        Add a command to the daemon queue, used by the communication daemon (communicator.py)
+    '''
+
+    retData = True
+
+    date = core_inst._utils.getEpoch()
+    conn = sqlite3.connect(core_inst.queueDB, timeout=30)
+    c = conn.cursor()
+    t = (command, data, date, responseID)
+    try:
+        c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t)
+        conn.commit()
+    except sqlite3.OperationalError:
+        retData = False
+        core_inst.daemonQueue()
+    events.event('queue_push', data = {'command': command, 'data': data}, onionr = core_inst.onionrInst)
+    conn.close()
+    return retData
+
+def daemon_queue_get_response(core_inst, responseID=''):
+    '''
+        Get a response sent by communicator to the API, by requesting to the API
+    '''
+    assert len(responseID) > 0
+    resp = core_inst._utils.localCommand('queueResponse/' + responseID)
+    return resp
+
+def clear_daemon_queue(core_inst):
+    '''
+        Clear the daemon queue (somewhat dangerous)
+    '''
+    conn = sqlite3.connect(core_inst.queueDB, timeout=30)
+    c = conn.cursor()
+
+    try:
+        c.execute('DELETE FROM commands;')
+        conn.commit()
+    except:
+        pass
+
+    conn.close()
+    events.event('queue_clear', onionr = core_inst.onionrInst)

onionr/coredb/keydb/__init__.py

@@ -0,0 +1 @@
+from . import addkeys, listkeys, removekeys, userinfo, transportinfo

onionr/coredb/keydb/addkeys.py

@@ -0,0 +1,70 @@
+import sqlite3
+import onionrevents as events, config
+def add_peer(core_inst, peerID, name=''):
+    '''
+        Adds a public key to the key database (misleading function name)
+    '''
+    if peerID in core_inst.listPeers() or peerID == core_inst._crypto.pubKey:
+        raise ValueError("specified id is already known")
+
+    # This function simply adds a peer to the DB
+    if not core_inst._utils.validatePubKey(peerID):
+        return False
+
+    events.event('pubkey_add', data = {'key': peerID}, onionr = core_inst.onionrInst)
+
+    conn = sqlite3.connect(core_inst.peerDB, timeout=30)
+    hashID = core_inst._crypto.pubKeyHashID(peerID)
+    c = conn.cursor()
+    t = (peerID, name, 'unknown', hashID, 0)
+
+    for i in c.execute("SELECT * FROM peers WHERE id = ?;", (peerID,)):
+        try:
+            if i[0] == peerID:
+                conn.close()
+                return False
+        except ValueError:
+            pass
+        except IndexError:
+            pass
+    c.execute('INSERT INTO peers (id, name, dateSeen, hashID, trust) VALUES(?, ?, ?, ?, ?);', t)
+    conn.commit()
+    conn.close()
+
+    return True
+
+def add_address(core_inst, address):
+    '''
+        Add an address to the address database (only tor currently)
+    '''
+
+    if type(address) is None or len(address) == 0:
+        return False
+    if core_inst._utils.validateID(address):
+        if address == config.get('i2p.ownAddr', None) or address == core_inst.hsAddress:
+            return False
+        conn = sqlite3.connect(core_inst.addressDB, timeout=30)
+        c = conn.cursor()
+        # check if address is in database
+        # this is safe to do because the address is validated above, but we strip some chars here too just in case
+        address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '')
+        for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)):
+            try:
+                if i[0] == address:
+                    conn.close()
+                    return False
+            except ValueError:
+                pass
+            except IndexError:
+                pass
+
+        t = (address, 1)
+        c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t)
+        conn.commit()
+        conn.close()
+
+        events.event('address_add', data = {'address': address}, onionr = core_inst.onionrInst)
+
+        return True
+    else:
+        return False

onionr/coredb/keydb/listkeys.py

@@ -0,0 +1,63 @@
+import sqlite3
+import logger
+def list_peers(core_inst, randomOrder=True, getPow=False, trust=0):
+    '''
+        Return a list of public keys (misleading function name)
+
+        randomOrder determines if the list should be in a random order
+        trust sets the minimum trust to list
+    '''
+    conn = sqlite3.connect(core_inst.peerDB, timeout=30)
+    c = conn.cursor()
+
+    payload = ''
+
+    if trust not in (0, 1, 2):
+        logger.error('Tried to select invalid trust.')
+        return
+
+    if randomOrder:
+        payload = 'SELECT * FROM peers WHERE trust >= ? ORDER BY RANDOM();'
+    else:
+        payload = 'SELECT * FROM peers WHERE trust >= ?;'
+
+    peerList = []
+
+    for i in c.execute(payload, (trust,)):
+        try:
+            if len(i[0]) != 0:
+                if getPow:
+                    peerList.append(i[0] + '-' + i[1])
+                else:
+                    peerList.append(i[0])
+        except TypeError:
+            pass
+
+    conn.close()
+
+    return peerList
+
+def list_adders(core_inst, randomOrder=True, i2p=True, recent=0):
+    '''
+        Return a list of transport addresses
+    '''
+    conn = sqlite3.connect(core_inst.addressDB, timeout=30)
+    c = conn.cursor()
+    if randomOrder:
+        addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();')
+    else:
+        addresses = c.execute('SELECT * FROM adders;')
+    addressList = []
+    for i in addresses:
+        if len(i[0].strip()) == 0:
+            continue
+        addressList.append(i[0])
+    conn.close()
+    testList = list(addressList) # create new list to iterate
+    for address in testList:
+        try:
+            if recent > 0 and (core_inst._utils.getEpoch() - core_inst.getAddressInfo(address, 'lastConnect')) > recent:
+                raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0
+        except TypeError:
+            addressList.remove(address)
+    return addressList

onionr/coredb/keydb/removekeys.py

@@ -0,0 +1,19 @@
+import sqlite3
+import onionrevents as events
+def remove_address(core_inst, address):
+    '''
+        Remove an address from the address database
+    '''
+
+    if core_inst._utils.validateID(address):
+        conn = sqlite3.connect(core_inst.addressDB, timeout=30)
+        c = conn.cursor()
+        t = (address,)
+        c.execute('Delete from adders where address=?;', t)
+        conn.commit()
+        conn.close()
+
+        events.event('address_remove', data = {'address': address}, onionr = core_inst.onionrInst)
+        return True
+    else:
+        return False

onionr/coredb/keydb/transportinfo.py

@@ -0,0 +1,53 @@
+import sqlite3
+def get_address_info(core_inst, address, info):
+    '''
+        Get info about an address from its database entry
+
+        address text, 0
+        type int, 1
+        knownPeer text, 2
+        speed int, 3
+        success int, 4
+        powValue    5
+        failure int 6
+        lastConnect 7
+        trust       8
+        introduced  9
+    '''
+
+    conn = sqlite3.connect(core_inst.addressDB, timeout=30)
+    c = conn.cursor()
+
+    command = (address,)
+    infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'powValue': 5, 'failure': 6, 'lastConnect': 7, 'trust': 8, 'introduced': 9}
+    info = infoNumbers[info]
+    iterCount = 0
+    retVal = ''
+
+    for row in c.execute('SELECT * FROM adders WHERE address=?;', command):
+        for i in row:
+            if iterCount == info:
+                retVal = i
+                break
+            else:
+                iterCount += 1
+    conn.close()
+
+    return retVal
+
+def set_address_info(core_inst, address, key, data):
+    '''
+        Update an address for a key
+    '''
+
+    conn = sqlite3.connect(core_inst.addressDB, timeout=30)
+    c = conn.cursor()
+
+    command = (data, address)
+
+    if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'failure', 'powValue', 'lastConnect', 'lastConnectAttempt', 'trust', 'introduced'):
+        raise Exception("Got invalid database key when setting address info")
+    else:
+        c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command)
+        conn.commit()
+    conn.close()

onionr/coredb/keydb/userinfo.py

@@ -0,0 +1,50 @@
+import sqlite3
+def get_user_info(core_inst, peer, info):
+    '''
+        Get info about a peer from their database entry
+
+        id text             0
+        name text,          1
+        adders text,        2
+        dateSeen not null,  3
+        trust int           4
+        hashID text         5
+    '''
+    conn = sqlite3.connect(core_inst.peerDB, timeout=30)
+    c = conn.cursor()
+
+    command = (peer,)
+    infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'trust': 4, 'hashID': 5}
+    info = infoNumbers[info]
+    iterCount = 0
+    retVal = ''
+
+    for row in c.execute('SELECT * FROM peers WHERE id=?;', command):
+        for i in row:
+            if iterCount == info:
+                retVal = i
+                break
+            else:
+                iterCount += 1
+
+    conn.close()
+
+    return retVal
+
+def set_peer_info(core_inst, peer, key, data):
+    '''
+        Update a peer for a key
+    '''
+
+    conn = sqlite3.connect(core_inst.peerDB, timeout=30)
+    c = conn.cursor()
+
+    command = (data, peer)
+
+    # TODO: validate key on whitelist
+    if key not in ('id', 'name', 'pubkey', 'forwardKey', 'dateSeen', 'trust'):
+        raise Exception("Got invalid database key when setting peer info")
+
+    c.execute('UPDATE peers SET ' + key + ' = ? WHERE id=?', command)
+    conn.commit()
+    conn.close()

onionr/onionrstorage.py

@@ -17,7 +17,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import core, sys, sqlite3, os, dbcreator
+import core, sys, sqlite3, os, dbcreator, onionrexceptions
 
 DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
 
@@ -94,4 +94,6 @@ def getData(coreInst, bHash):
             retData = block.read()
     else:
         retData = _dbFetch(coreInst, bHash)
+        if retData is None:
+            raise onionrexceptions.NoDataAvailable("Block data for %s is not available" % [bHash])
     return retData

onionr/static-data/default-plugins/esoteric/controlapi.py

@@ -22,13 +22,13 @@ from flask import Response, request, redirect, Blueprint, send_from_directory
 import core
 
 core_inst = core.Core()
-flask_blueprint = Blueprint('clandestine_control', __name__)
+flask_blueprint = Blueprint('esoteric_control', __name__)
 
-@flask_blueprint.route('/clandestine/ping')
+@flask_blueprint.route('/esoteric/ping')
 def ping():
     return 'pong!'
 
-@flask_blueprint.route('/clandestine/send/<peer>', methods=['POST'])
+@flask_blueprint.route('/esoteric/send/<peer>', methods=['POST'])
 def send_message(peer):
     data = request.get_json(force=True)
     core_inst.keyStore.refresh()
@@ -40,14 +40,14 @@ def send_message(peer):
     core_inst.keyStore.flush()
     return Response('success')
 
-@flask_blueprint.route('/clandestine/gets/<peer>')
+@flask_blueprint.route('/esoteric/gets/<peer>')
 def get_sent(peer):
     sent = core_inst.keyStore.get('s' + peer)
     if sent is None:
         sent = []
     return Response(json.dumps(sent))
 
-@flask_blueprint.route('/clandestine/addrec/<peer>', methods=['POST'])
+@flask_blueprint.route('/esoteric/addrec/<peer>', methods=['POST'])
 def add_rec(peer):
     data = request.get_json(force=True)
     core_inst.keyStore.refresh()
@@ -59,7 +59,7 @@ def add_rec(peer):
     core_inst.keyStore.flush()
     return Response('success')
 
-@flask_blueprint.route('/clandestine/getrec/<peer>')
+@flask_blueprint.route('/esoteric/getrec/<peer>')
 def get_messages(peer):
     core_inst.keyStore.refresh()
     existing = core_inst.keyStore.get('r' + peer)

onionr/static-data/default-plugins/esoteric/info.json

@@ -1,5 +1,5 @@
 {
-    "name" : "clandestine",
+    "name" : "esoteric",
     "version" : "1.0",
     "author" : "onionr"
 }

onionr/static-data/default-plugins/esoteric/main.py

@@ -24,7 +24,7 @@ locale.setlocale(locale.LC_ALL, '')
 import onionrservices, logger
 from onionrservices import bootstrapservice
 
-plugin_name = 'clandestine'
+plugin_name = 'esoteric'
 PLUGIN_VERSION = '0.0.0'
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
 import controlapi, peerserver
@@ -36,7 +36,7 @@ def exit_with_error(text=''):
         logger.error(text)
     sys.exit(1)
 
-class Clandestine:
+class Esoteric:
     def __init__(self, pluginapi):
         self.myCore = pluginapi.get_core()
         self.peer = None
@@ -58,7 +58,7 @@ class Clandestine:
                     message += '\n'
             except EOFError:
                 message = json.dumps({'m': message, 't': self.myCore._utils.getEpoch()})
-                print(self.myCore._utils.doPostRequest('http://%s/clandestine/sendto' % (self.transport,), port=self.socks, data=message))
+                print(self.myCore._utils.doPostRequest('http://%s/esoteric/sendto' % (self.transport,), port=self.socks, data=message))
                 message = ''
             except KeyboardInterrupt:
                 self.shutdown = True
@@ -89,6 +89,6 @@ def on_init(api, data = None):
     '''
 
     pluginapi = api
-    chat = Clandestine(pluginapi)
+    chat = Esoteric(pluginapi)
-    api.commands.register(['clandestine'], chat.create)
+    api.commands.register(['esoteric'], chat.create)
     return

onionr/static-data/default-plugins/esoteric/peerserver.py

@@ -21,7 +21,7 @@ import sys, os, json
 import core
 from flask import Response, request, redirect, Blueprint, abort, g
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
-direct_blueprint = Blueprint('clandestine', __name__)
+direct_blueprint = Blueprint('esoteric', __name__)
 core_inst = core.Core()
 
 storage_dir = core_inst.dataDir
@@ -35,11 +35,11 @@ def request_setup():
     g.host = host
     g.peer = core_inst.keyStore.get('dc-' + g.host)
 
-@direct_blueprint.route('/clandestine/ping')
+@direct_blueprint.route('/esoteric/ping')
 def pingdirect():
     return 'pong!'
 
-@direct_blueprint.route('/clandestine/sendto', methods=['POST', 'GET'])
+@direct_blueprint.route('/esoteric/sendto', methods=['POST', 'GET'])
 def sendto():
     try:
         msg = request.get_json(force=True)
@@ -47,9 +47,9 @@ def sendto():
         msg = ''
     else:
         msg = json.dumps(msg)
-        core_inst._utils.localCommand('/clandestine/addrec/%s' % (g.peer,), post=True, postData=msg)
+        core_inst._utils.localCommand('/esoteric/addrec/%s' % (g.peer,), post=True, postData=msg)
     return Response('success')
 
-@direct_blueprint.route('/clandestine/poll')
+@direct_blueprint.route('/esoteric/poll')
 def poll_chat():
-    return Response(core_inst._utils.localCommand('/clandestine/gets/%s' % (g.peer,)))
+    return Response(core_inst._utils.localCommand('/esoteric/gets/%s' % (g.peer,)))

onionr/static-data/default-plugins/flow/main.py

@@ -61,6 +61,7 @@ class OnionrFlow:
                     self.flowRunning = False
                 expireTime = self.myCore._utils.getEpoch() + 43200
                 if len(message) > 0:
+                    logger.info('Inserting message as block...', terminal=True)
                     self.myCore.insertBlock(message, header='txt', expire=expireTime, meta={'ch': self.channel})
 
         logger.info("Flow is exiting, goodbye", terminal=True)

onionr/static-data/www/private/index.html

@@ -26,7 +26,7 @@
             <label>Open Site: <input type='text' id='siteViewer' placeholder='Site Hash'> <button id='openSite' class='btn primaryBtn openSiteBtn'>Open Onionr Site</button></label>
             <br>
             <br><br><a class='idLink' href='/mail/'>Mail</a> - <a class='idLink' href='/friends/'>Friend Manager</a> - <a class='idLink' href='/board/'>Circle</a> - 
-                    <a class='idLink' href='/clandestine/'>Clandestine</a>
+                    <a class='idLink' href='/esoteric/'>Esoteric</a>
             <br><br><hr>
             <details class='configArea'>
                 <summary><b>Edit Configuration</b></summary>