added mixmate to improve base routing

src/communicatorutils/daemonqueuehandler.py

@@ -1,9 +1,15 @@
-'''
-    Onionr - P2P Anonymous Storage Network
+"""Onionr - P2P Anonymous Storage Network.
 
-    Handle daemon queue commands in the communicator
-'''
-'''
+Handle daemon queue commands in the communicator
+"""
+import logger
+from onionrplugins import onionrevents as events
+from onionrutils import localcommand
+from coredb import daemonqueue
+import filepaths
+from . import restarttor
+from communicatorutils.uploadblocks import mixmate
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,13 +22,9 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import logger
-from onionrplugins import onionrevents as events
-from onionrutils import localcommand
-from coredb import daemonqueue
-import filepaths
-from . import restarttor
+"""
+
+
 def handle_daemon_commands(comm_inst):
     cmd = daemonqueue.daemon_queue()
     response = ''
@@ -62,6 +64,13 @@ def handle_daemon_commands(comm_inst):
                     i.count = (i.frequency - 1)
         elif cmd[0] == 'uploadBlock':
             comm_inst.blocksToUpload.append(cmd[1])
+        elif cmd[0] == 'uploadEvent':
+            try:
+                mixmate.block_mixer(comm_inst.blocksToUpload, cmd[1])
+            except ValueError:
+                pass
+            else:
+                localcommand.local_command('/waitforshare/' + cmd[1], post=True, maxWait=5)
         else:
             logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],))
 

src/communicatorutils/downloadblocks/__init__.py

@@ -1,8 +1,11 @@
-'''
+"""
     Onionr - Private P2P Communication
 
     Download blocks using the communicator instance
-'''
+"""
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from communicator import OnionrCommunicatorDaemon
 import onionrexceptions
 import logger
 import onionrpeers
@@ -12,12 +15,13 @@ from communicator import onlinepeers
 from onionrutils import blockmetadata
 from onionrutils import validatemetadata
 from coredb import blockmetadb
+from coredb import daemonqueue
 import onionrcrypto
 import onionrstorage
 from onionrblocks import onionrblacklist
 from onionrblocks import storagecounter
 from . import shoulddownload
-'''
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -30,11 +34,11 @@ from . import shoulddownload
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
 
 
 def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
-    '''Use communicator instance to download blocks in the comms's queue'''
+    """Use communicator instance to download blocks in the comms's queue"""
     blacklist = onionrblacklist.OnionrBlackList()
     storage_counter = storagecounter.StorageCounter()
     LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
@@ -109,6 +113,7 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
                             removeFromQueue = False
                         else:
                             blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db
+                            daemonqueue.daemon_queue_add('uploadEvent', blockHash)
                             blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
                     else:
                         logger.warn('POW failed for block %s.' % (blockHash,))

src/communicatorutils/uploadblocks/__init__.py

@@ -27,6 +27,7 @@ from onionrutils import localcommand, stringvalidators, basicrequests
 from communicator import onlinepeers
 import onionrcrypto
 from . import sessionmanager
+from . import mixmate
 
 def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
     """Accepts a communicator instance and uploads blocks from its upload queue"""

src/communicatorutils/uploadblocks/mixmate/__init__.py

@@ -0,0 +1,51 @@
+"""Onionr - Private P2P Communication.
+
+Delay block uploads, optionally mixing them together
+"""
+import time
+from typing import List
+
+import onionrtypes
+from onionrblocks import onionrblockapi
+
+from .pool import UploadPool
+from .pool import PoolFullException
+
+from etc import onionrvalues
+
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+upload_pool = UploadPool(4)
+
+
+def block_mixer(upload_list: List[onionrtypes.BlockHash],
+                block_to_mix: onionrtypes.BlockHash):
+    """Delay and mix block inserts.
+
+    Take a block list and a received/created block and add it
+    to the said block list
+    """
+    bl = onionrblockapi.Block(block_to_mix)
+    if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE:
+        raise ValueError
+
+    try:
+        # add the new block to pool
+        upload_pool.add_to_pool(block_to_mix)
+    except PoolFullException:
+        # If the pool is full, move into upload queue
+        upload_list.extend(upload_pool.get_pool())
+        # then finally begin new pool with new block
+        upload_pool.add_to_pool(block_to_mix)

src/communicatorutils/uploadblocks/mixmate/pool.py

@@ -0,0 +1,69 @@
+"""Onionr - Private P2P Communication.
+
+Upload pool
+"""
+from typing import List
+
+import onionrutils
+import onionrtypes
+from onionrcrypto import cryptoutils
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+
+class PoolFullException(Exception):
+    """For when the UploadPool is full.
+
+    Raise when a new hash is attempted to be added
+    """
+
+
+class PoolNotReady(Exception):
+    """Raise when UploadPool pool access is attempted without it being full."""
+
+
+class AlreadyInPool(Exception):
+    """Raise when a hash already in pool is attempted to be added again."""
+
+
+class UploadPool:
+    """Upload pool for mixing blocks together and delaying uploads."""
+
+    def __init__(self, pool_size: int):
+        """Create a new pool with a specified max size.
+
+        Uses private var and getter to avoid direct adding
+        """
+        self._pool: List[onionrtypes.BlockHash] = []
+        self._pool_size = pool_size
+        self.birthday = onionrutils.epoch.get_epoch()
+
+    def add_to_pool(self, item: List[onionrtypes.BlockHash]):
+        """Add a new hash to the pool. Raise PoolFullException if full."""
+        if len(self._pool) >= self._pool_size:
+            raise PoolFullException
+        if not onionrutils.stringvalidators.validate_hash(item):
+            raise ValueError
+        self._pool.append(item)
+
+    def get_pool(self) -> List[onionrtypes.BlockHash]:
+        """Get the hash pool in secure random order."""
+        if len(self._pool) != self._pool_size:
+            raise PoolNotReady
+        final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(list(self._pool))
+
+        self._pool.clear()
+        self.birthday = onionrutils.epoch.get_epoch()
+        return final_pool

src/communicatorutils/uploadblocks/session.py

@@ -1,9 +1,13 @@
-"""
-    Onionr - Private P2P Communication
+"""Onionr - Private P2P Communication.
 
-    Virtual upload "sessions" for blocks
+Virtual upload "sessions" for blocks
 """
-from __future__ import annotations
+from typing import Union
+
+from onionrutils import stringvalidators
+from onionrutils import bytesconverter
+from onionrutils import epoch
+from utils import reconstructhash
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -18,21 +22,19 @@ from __future__ import annotations
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """
-from typing import Union
 
-from onionrutils import stringvalidators
-from onionrutils import bytesconverter
-from onionrutils import epoch
-from utils import reconstructhash
 
 class UploadSession:
-    """Manages statistics for an Onionr block upload session
-    
-    accepting a block hash (incl. unpadded) as an argument""" 
+    """Manage statistics for an Onionr block upload session.
+
+    accept a block hash (incl. unpadded) as an argument
+    """
+
     def __init__(self, block_hash: Union[str, bytes]):
         block_hash = bytesconverter.bytes_to_str(block_hash)
         block_hash = reconstructhash.reconstruct_hash(block_hash)
-        if not stringvalidators.validate_hash(block_hash): raise ValueError
+        if not stringvalidators.validate_hash(block_hash):
+            raise ValueError
 
         self.start_time = epoch.get_epoch()
         self.block_hash = reconstructhash.deconstruct_hash(block_hash)
@@ -40,15 +42,15 @@ class UploadSession:
         self.total_success_count: int = 0
         self.peer_fails = {}
         self.peer_exists = {}
-    
+
     def fail_peer(self, peer):
         try:
             self.peer_fails[peer] += 1
         except KeyError:
             self.peer_fails[peer] = 0
-    
+
     def fail(self):
         self.total_fail_count += 1
-    
+
     def success(self):
         self.total_success_count += 1