added mixmate to improve base routing

src/bigbrother/ministry/ofexec.py

@@ -57,6 +57,8 @@ def block_exec(event, info):
                        ]
     home = identifyhome.identify_home()
 
+    code_b64 = base64.b64encode(info[0].co_code).decode()
+
     for source in whitelisted_code:
         if info[0].co_filename.endswith(source):
             return
@@ -64,7 +66,6 @@ def block_exec(event, info):
     if home + 'plugins/' in info[0].co_filename:
         return
 
-    code_b64 = base64.b64encode(info[0].co_code).decode()
     logger.warn('POSSIBLE EXPLOIT DETECTED, SEE LOGS', terminal=True)
     logger.warn('POSSIBLE EXPLOIT DETECTED: ' + info[0].co_filename)
     logger.warn('Prevented exec/eval. Report this with the sample below')

src/communicatorutils/daemonqueuehandler.py

@@ -1,9 +1,15 @@
-'''
+"""Onionr - P2P Anonymous Storage Network.
-    Onionr - P2P Anonymous Storage Network
 
-    Handle daemon queue commands in the communicator
+Handle daemon queue commands in the communicator
-'''
+"""
-'''
+import logger
+from onionrplugins import onionrevents as events
+from onionrutils import localcommand
+from coredb import daemonqueue
+import filepaths
+from . import restarttor
+from communicatorutils.uploadblocks import mixmate
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,13 +22,9 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
-import logger
+
-from onionrplugins import onionrevents as events
+
-from onionrutils import localcommand
-from coredb import daemonqueue
-import filepaths
-from . import restarttor
 def handle_daemon_commands(comm_inst):
     cmd = daemonqueue.daemon_queue()
     response = ''
@@ -62,6 +64,13 @@ def handle_daemon_commands(comm_inst):
                     i.count = (i.frequency - 1)
         elif cmd[0] == 'uploadBlock':
             comm_inst.blocksToUpload.append(cmd[1])
+        elif cmd[0] == 'uploadEvent':
+            try:
+                mixmate.block_mixer(comm_inst.blocksToUpload, cmd[1])
+            except ValueError:
+                pass
+            else:
+                localcommand.local_command('/waitforshare/' + cmd[1], post=True, maxWait=5)
         else:
             logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],))
 

src/communicatorutils/downloadblocks/__init__.py

@@ -1,8 +1,11 @@
-'''
+"""
     Onionr - Private P2P Communication
 
     Download blocks using the communicator instance
-'''
+"""
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from communicator import OnionrCommunicatorDaemon
 import onionrexceptions
 import logger
 import onionrpeers
@@ -12,12 +15,13 @@ from communicator import onlinepeers
 from onionrutils import blockmetadata
 from onionrutils import validatemetadata
 from coredb import blockmetadb
+from coredb import daemonqueue
 import onionrcrypto
 import onionrstorage
 from onionrblocks import onionrblacklist
 from onionrblocks import storagecounter
 from . import shoulddownload
-'''
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -30,11 +34,11 @@ from . import shoulddownload
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
 
 
 def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
-    '''Use communicator instance to download blocks in the comms's queue'''
+    """Use communicator instance to download blocks in the comms's queue"""
     blacklist = onionrblacklist.OnionrBlackList()
     storage_counter = storagecounter.StorageCounter()
     LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
@@ -109,6 +113,7 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
                             removeFromQueue = False
                         else:
                             blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db
+                            daemonqueue.daemon_queue_add('uploadEvent', blockHash)
                             blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
                     else:
                         logger.warn('POW failed for block %s.' % (blockHash,))

src/communicatorutils/uploadblocks/__init__.py

@@ -27,6 +27,7 @@ from onionrutils import localcommand, stringvalidators, basicrequests
 from communicator import onlinepeers
 import onionrcrypto
 from . import sessionmanager
+from . import mixmate
 
 def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
     """Accepts a communicator instance and uploads blocks from its upload queue"""

src/communicatorutils/uploadblocks/mixmate/__init__.py

@@ -0,0 +1,51 @@
+"""Onionr - Private P2P Communication.
+
+Delay block uploads, optionally mixing them together
+"""
+import time
+from typing import List
+
+import onionrtypes
+from onionrblocks import onionrblockapi
+
+from .pool import UploadPool
+from .pool import PoolFullException
+
+from etc import onionrvalues
+
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+upload_pool = UploadPool(4)
+
+
+def block_mixer(upload_list: List[onionrtypes.BlockHash],
+                block_to_mix: onionrtypes.BlockHash):
+    """Delay and mix block inserts.
+
+    Take a block list and a received/created block and add it
+    to the said block list
+    """
+    bl = onionrblockapi.Block(block_to_mix)
+    if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE:
+        raise ValueError
+
+    try:
+        # add the new block to pool
+        upload_pool.add_to_pool(block_to_mix)
+    except PoolFullException:
+        # If the pool is full, move into upload queue
+        upload_list.extend(upload_pool.get_pool())
+        # then finally begin new pool with new block
+        upload_pool.add_to_pool(block_to_mix)

src/communicatorutils/uploadblocks/mixmate/pool.py

@@ -0,0 +1,69 @@
+"""Onionr - Private P2P Communication.
+
+Upload pool
+"""
+from typing import List
+
+import onionrutils
+import onionrtypes
+from onionrcrypto import cryptoutils
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+
+class PoolFullException(Exception):
+    """For when the UploadPool is full.
+
+    Raise when a new hash is attempted to be added
+    """
+
+
+class PoolNotReady(Exception):
+    """Raise when UploadPool pool access is attempted without it being full."""
+
+
+class AlreadyInPool(Exception):
+    """Raise when a hash already in pool is attempted to be added again."""
+
+
+class UploadPool:
+    """Upload pool for mixing blocks together and delaying uploads."""
+
+    def __init__(self, pool_size: int):
+        """Create a new pool with a specified max size.
+
+        Uses private var and getter to avoid direct adding
+        """
+        self._pool: List[onionrtypes.BlockHash] = []
+        self._pool_size = pool_size
+        self.birthday = onionrutils.epoch.get_epoch()
+
+    def add_to_pool(self, item: List[onionrtypes.BlockHash]):
+        """Add a new hash to the pool. Raise PoolFullException if full."""
+        if len(self._pool) >= self._pool_size:
+            raise PoolFullException
+        if not onionrutils.stringvalidators.validate_hash(item):
+            raise ValueError
+        self._pool.append(item)
+
+    def get_pool(self) -> List[onionrtypes.BlockHash]:
+        """Get the hash pool in secure random order."""
+        if len(self._pool) != self._pool_size:
+            raise PoolNotReady
+        final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(list(self._pool))
+
+        self._pool.clear()
+        self.birthday = onionrutils.epoch.get_epoch()
+        return final_pool

src/communicatorutils/uploadblocks/session.py

@@ -1,9 +1,13 @@
-"""
+"""Onionr - Private P2P Communication.
-    Onionr - Private P2P Communication
 
-    Virtual upload "sessions" for blocks
+Virtual upload "sessions" for blocks
 """
-from __future__ import annotations
+from typing import Union
+
+from onionrutils import stringvalidators
+from onionrutils import bytesconverter
+from onionrutils import epoch
+from utils import reconstructhash
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -18,21 +22,19 @@ from __future__ import annotations
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """
-from typing import Union
 
-from onionrutils import stringvalidators
-from onionrutils import bytesconverter
-from onionrutils import epoch
-from utils import reconstructhash
 
 class UploadSession:
-    """Manages statistics for an Onionr block upload session
+    """Manage statistics for an Onionr block upload session.
-    
+
-    accepting a block hash (incl. unpadded) as an argument""" 
+    accept a block hash (incl. unpadded) as an argument
+    """
+
     def __init__(self, block_hash: Union[str, bytes]):
         block_hash = bytesconverter.bytes_to_str(block_hash)
         block_hash = reconstructhash.reconstruct_hash(block_hash)
-        if not stringvalidators.validate_hash(block_hash): raise ValueError
+        if not stringvalidators.validate_hash(block_hash):
+            raise ValueError
 
         self.start_time = epoch.get_epoch()
         self.block_hash = reconstructhash.deconstruct_hash(block_hash)
@@ -40,15 +42,15 @@ class UploadSession:
         self.total_success_count: int = 0
         self.peer_fails = {}
         self.peer_exists = {}
-    
+
     def fail_peer(self, peer):
         try:
             self.peer_fails[peer] += 1
         except KeyError:
             self.peer_fails[peer] = 0
-    
+
     def fail(self):
         self.total_fail_count += 1
-    
+
     def success(self):
         self.total_success_count += 1

src/etc/onionrvalues.py

@@ -52,6 +52,9 @@ DEFAULT_EXPIRE = 2592000
 # Metadata header section length limits, in bytes
 BLOCK_METADATA_LENGTHS = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'pow': 1000, 'encryptType': 4, 'expire': 14}
 
+# Pool Eligibility Max Age
+BLOCK_POOL_MAX_AGE = 300
+
 """Public key that signs MOTD messages shown in the web UI"""
 MOTD_SIGN_KEY = "TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ"
 

src/httpapi/miscclientapi/endpoints.py

@@ -57,7 +57,7 @@ class PrivateEndpoints:
             # Responses from the daemon. TODO: change to direct var access instead of http endpoint
             client_api.queueResponse[name] = request.form['data']
             return Response('success')
-        
+
         @private_endpoints_bp.route('/queueResponse/<name>')
         def queueResponse(name):
             # Fetch a daemon queue response
@@ -72,7 +72,7 @@ class PrivateEndpoints:
                 return resp, 404
             else:
                 return resp
-            
+
         @private_endpoints_bp.route('/ping')
         def ping():
             # Used to check if client api is working
@@ -102,24 +102,24 @@ class PrivateEndpoints:
         def restart_clean():
             subprocess.Popen([SCRIPT_NAME, 'restart'])
             return Response("bye")
-        
+
         @private_endpoints_bp.route('/gethidden')
         def get_hidden_blocks():
             return Response('\n'.join(client_api.publicAPI.hideBlocks))
-        
+
         @private_endpoints_bp.route('/getstats')
         def getStats():
             # returns node stats
             while True:
-                try:    
+                try:
                     return Response(client_api._too_many.get(SerializedData).get_stats())
                 except AttributeError as e:
                     pass
-        
+
         @private_endpoints_bp.route('/getuptime')
         def showUptime():
             return Response(str(client_api.getUptime()))
-        
+
         @private_endpoints_bp.route('/getActivePubkey')
         def getActivePubkey():
             return Response(pub_key)
@@ -132,7 +132,7 @@ class PrivateEndpoints:
         def getHumanReadable(name):
             name = unpaddedbase32.repad(bytesconverter.str_to_bytes(name))
             return Response(mnemonickeys.get_human_readable_ID(name))
-        
+
         @private_endpoints_bp.route('/getBase32FromHumanReadable/<words>')
         def get_base32_from_human_readable(words):
             return Response(bytesconverter.bytes_to_str(mnemonickeys.get_base32(words)))
@@ -144,3 +144,4 @@ class PrivateEndpoints:
         @private_endpoints_bp.route('/setonboarding', methods=['POST'])
         def set_onboarding():
             return Response(config.onboarding.set_config_from_onboarding(request.get_json()))
+

src/httpapi/miscpublicapi/upload.py

@@ -3,10 +3,14 @@
 
     Accept block uploads to the public API server
 '''
+from gevent import threading
+
 import sys
 from flask import Response
 from flask import abort
 
+from onionrutils import localcommand
+from coredb import daemonqueue
 from onionrblocks import blockimporter
 import onionrexceptions
 import logger
@@ -31,9 +35,12 @@ def accept_upload(request):
     """Accept uploaded blocks to our public Onionr protocol API server"""
     resp = 'failure'
     data = request.get_data()
+    b_hash = ''
     if sys.getsizeof(data) < 100000000:
         try:
-            if blockimporter.import_block_from_data(data):
+            b_hash = blockimporter.import_block_from_data(data)
+            if b_hash:
+                daemonqueue.daemon_queue_add('uploadEvent', b_hash)
                 resp = 'success'
             else:
                 resp = 'failure'

src/onionrblocks/insert.py

@@ -82,7 +82,7 @@ def insert_block(data: Union[str, bytes], header: str = 'txt',
         our_pub_key = bytesconverter.bytes_to_str(crypto.cryptoutils.get_pub_key_from_priv(our_private_key))
 
     use_subprocess = powchoice.use_subprocess(config)
-    
+
     retData = False
 
     if type(data) is None:
@@ -195,20 +195,15 @@ def insert_block(data: Union[str, bytes], header: str = 'txt',
             retData = False
         else:
             # Tell the api server through localCommand to wait for the daemon to upload this block to make statistical analysis more difficult
-            if not is_offline or localcommand.local_command('/ping', maxWait=10) == 'pong!':
+            coredb.daemonqueue.daemon_queue_add('uploadEvent', retData)
-                if config.get('general.security_level', 1) == 0:
-                    localcommand.local_command('/waitforshare/' + retData, post=True, maxWait=5)
-                coredb.daemonqueue.daemon_queue_add('uploadBlock', retData)
-            else:
-                pass
             coredb.blockmetadb.add.add_to_block_DB(retData, selfInsert=True, dataSaved=True)
 
             if expire is None:
-                coredb.blockmetadb.update_block_info(retData, 'expire', 
+                coredb.blockmetadb.update_block_info(retData, 'expire',
                                                      createTime + onionrvalues.DEFAULT_EXPIRE)
             else:
                 coredb.blockmetadb.update_block_info(retData, 'expire', expire)
-            
+
             blockmetadata.process_block_metadata(retData)
 
     if retData != False:

src/runtests/__init__.py

@@ -7,6 +7,7 @@ import logger
 from onionrutils import epoch
 
 from . import uicheck, inserttest, stresstest
+from . import ownnode
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -24,7 +25,10 @@ from . import uicheck, inserttest, stresstest
 
 RUN_TESTS = [uicheck.check_ui,
              inserttest.insert_bin_test,
-             stresstest.stress_test_block_insert]
+             ownnode.test_tor_adder,
+             ownnode.test_own_node,
+             stresstest.stress_test_block_insert
+             ]
 
 
 class OnionrRunTestManager:
@@ -35,10 +39,11 @@ class OnionrRunTestManager:
     def run_tests(self):
         cur_time = epoch.get_epoch()
         logger.info(f"Doing runtime tests at {cur_time}")
+
         try:
             for i in RUN_TESTS:
                 last = i
                 i(self)
-                logger.info(last.__name__ + " passed")
+                logger.info("[RUNTIME TEST] " + last.__name__ + " passed")
-        except ValueError:
+        except (ValueError, AttributeError):
             logger.error(last.__name__ + ' failed')

src/runtests/ownnode.py

@@ -0,0 +1,53 @@
+"""Onionr - Private P2P Communication.
+
+Test own Onionr node as it is running
+"""
+import config
+from onionrutils import basicrequests
+from utils import identifyhome
+from utils import gettransports
+import logger
+from onionrutils import localcommand
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+
+def test_own_node(test_manager):
+    socks_port = localcommand.local_command('/gettorsocks')
+    if config.get('general.security_level', 0) > 0:
+        return
+    own_tor_address = gettransports.get()[0]
+    print(socks_port)
+    if 'this is an onionr node' \
+            not in basicrequests.do_get_request(own_tor_address,
+                                                port=socks_port).lower():
+        logger.warn('Own node not reachable in test')
+        raise ValueError
+
+
+def test_tor_adder(test_manager):
+    if config.get('general.security_level', 0) > 0:
+        return
+    with open(identifyhome.identify_home() + 'hs/hostname', 'r') as hs:
+        hs = hs.read().strip()
+    if not hs:
+        logger.error('No Tor node address created yet')
+        raise ValueError('No Tor node address created yet')
+
+    if hs not in gettransports.get():
+        print(hs in gettransports.get(), 'meme')
+        logger.error('gettransports Tor not same as file: %s %s' %
+                     (hs, gettransports.get()))
+        raise ValueError('gettransports Tor not same as file')

src/runtests/stresstest.py

@@ -6,6 +6,7 @@ import coredb
 from onionrutils import epoch
 
 def stress_test_block_insert(testmanager):
+    return
     start = epoch.get_epoch()
     count = 100
     max_insert_speed = 120

src/utils/gettransports.py

@@ -1,17 +1,20 @@
-import filepaths, time
+from gevent import time
+
+import filepaths
 
 files = [filepaths.tor_hs_address_file]
 
+
 def get():
-        transports = []
+    transports = []
-        for file in files:
+    for file in files:
-                try:
+        try:
-                        with open(file, 'r') as transport_file:
+            with open(file, 'r') as transport_file:
-                                transports.append(transport_file.read().strip())
+                transports.append(transport_file.read().strip())
-                except FileNotFoundError:
+        except FileNotFoundError:
-                        pass
+            pass
-                else:
-                        break
         else:
-                time.sleep(1)
+            break
-        return list(transports)
+    else:
+        time.sleep(1)
+    return list(transports)