work on block insertion mixing

src/communicatorutils/daemonqueuehandler/__init__.py

@@ -5,10 +5,12 @@ Handle daemon queue commands in the communicator
 import logger
 from onionrplugins import onionrevents as events
 from onionrutils import localcommand
+from communicatorutils.uploadblocks import mixmate
 from coredb import daemonqueue
 import filepaths
-from . import restarttor
+
-from communicatorutils.uploadblocks import mixmate
+from .. import restarttor
+
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -29,11 +31,12 @@ def handle_daemon_commands(comm_inst):
     cmd = daemonqueue.daemon_queue()
     response = ''
     if cmd is not False:
-        events.event('daemon_command', data = {'cmd' : cmd})
+        events.event('daemon_command', data={'cmd': cmd})
         if cmd[0] == 'shutdown':
             comm_inst.shutdown = True
         elif cmd[0] == 'runtimeTest':
-            comm_inst.shared_state.get_by_string("OnionrRunTestManager").run_tests()
+            comm_inst.shared_state.get_by_string(
+                "OnionrRunTestManager").run_tests()
         elif cmd[0] == 'remove_from_insert_list':
             try:
                 comm_inst.generating_blocks.remove(cmd[1])
@@ -44,7 +47,7 @@ def handle_daemon_commands(comm_inst):
                 comm_inst.announce(cmd[1])
             else:
                 logger.debug("No nodes connected. Will not introduce node.")
-        elif cmd[0] == 'runCheck': # deprecated
+        elif cmd[0] == 'runCheck':  # deprecated
             logger.debug('Status check; looks good.')
             open(filepaths.run_check_file + '.runcheck', 'w+').close()
         elif cmd[0] == 'connectedPeers':
@@ -65,18 +68,21 @@ def handle_daemon_commands(comm_inst):
         elif cmd[0] == 'uploadBlock':
             comm_inst.blocksToUpload.append(cmd[1])
         elif cmd[0] == 'uploadEvent':
+            localcommand.local_command('/waitforshare/' + cmd[1], post=True,
+                                       maxWait=5)
             try:
                 mixmate.block_mixer(comm_inst.blocksToUpload, cmd[1])
             except ValueError:
-                pass
+                comm_inst.blocksToUpload.append(cmd[1])
-            else:
-                localcommand.local_command('/waitforshare/' + cmd[1], post=True, maxWait=5)
         else:
-            logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],))
+            logger.debug(
+                'Received daemon queue cmd with no handler: %s' % (cmd[0],))
 
         if cmd[0] not in ('', None):
             if response != '':
-                localcommand.local_command('queueResponseAdd/' + cmd[4], post=True, postData={'data': response})
+                localcommand.local_command('queueResponseAdd/' + cmd[4],
+                                           post=True,
+                                           postData={'data': response})
         response = ''
 
     comm_inst.decrementThreadCount('handle_daemon_commands')

src/communicatorutils/uploadblocks/__init__.py

@@ -15,7 +15,6 @@ import onionrcrypto
 from communicator import onlinepeers
 if TYPE_CHECKING:
     from communicator import OnionrCommunicatorDaemon
-
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by

src/communicatorutils/uploadblocks/mixmate/__init__.py

@@ -38,6 +38,14 @@ def block_mixer(upload_list: List[onionrtypes.BlockHash],
     to the said block list
     """
     bl = onionrblockapi.Block(block_to_mix)
+
+    try:
+        bl.bmetadata['dly']
+    except (KeyError, TypeError):
+        pass
+    else:
+        raise ValueError
+
     if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE:
         raise ValueError
 

src/communicatorutils/uploadblocks/sessionmanager.py

@@ -106,7 +106,10 @@ class BlockUploadSessionManager:
             if (sess.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
                 sessions_to_delete.append(sess)
         for sess in sessions_to_delete:
-            self.sessions.remove(session)
+            try:
+                self.sessions.remove(session)
+            except ValueError:
+                pass
             # TODO cleanup to one round of search
             # Remove the blocks from the sessions, upload list,
             # and waitforshare list

src/etc/onionrvalues.py

@@ -43,6 +43,7 @@ DATABASE_LOCK_TIMEOUT = 60
 
 # Block creation anonymization requirements
 MIN_BLOCK_UPLOAD_PEER_PERCENT = 0.1
+MIN_SHARE_WAIT_DELAY_SECS = 5
 
 # Begin OnionrValues migrated values
 """Make announce take a few seconds (on average) to compute to discourage excessive node announcements"""

src/httpapi/miscclientapi/endpoints.py

@@ -1,9 +1,33 @@
-'''
+"""Onionr - Private P2P Communication.
-    Onionr - Private P2P Communication
 
-    Misc client API endpoints too small to need their own file and that need access to the client api inst
+Misc client API endpoints too small to need their own file
-'''
+and that need access to the client api inst
-'''
+"""
+from typing import TYPE_CHECKING
+from secrets import randbelow
+
+import os
+import subprocess
+
+from flask import Response, Blueprint, request, send_from_directory, abort
+from gevent import spawn
+from gevent import sleep
+import unpaddedbase32
+
+from httpapi import apiutils
+import logger
+import onionrcrypto
+import config
+from netcontroller import NetController
+from serializeddata import SerializedData
+from onionrutils import mnemonickeys
+from onionrutils import bytesconverter
+from etc import onionrvalues
+from utils import reconstructhash
+from onionrcommands import restartonionr
+if TYPE_CHECKING:
+    from onionrtypes import BlockHash
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,26 +40,14 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
-import os
-import subprocess
-
-from flask import Response, Blueprint, request, send_from_directory, abort
-import unpaddedbase32
-
-from httpapi import apiutils
-import onionrcrypto, config
-from netcontroller import NetController
-from serializeddata import SerializedData
-from onionrutils import mnemonickeys
-from onionrutils import bytesconverter
-from etc import onionrvalues
-from utils import reconstructhash
-from onionrcommands import restartonionr
 
 pub_key = onionrcrypto.pub_key.replace('=', '')
 
-SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + f'/../../../{onionrvalues.SCRIPT_NAME}'
+SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + \
+              f'/../../../{onionrvalues.SCRIPT_NAME}'
+
+
 
 class PrivateEndpoints:
     def __init__(self, client_api):
@@ -84,12 +96,30 @@ class PrivateEndpoints:
 
         @private_endpoints_bp.route('/waitforshare/<name>', methods=['post'])
         def waitforshare(name):
-            '''Used to prevent the **public** api from sharing blocks we just created'''
+            """Prevent the **public** api from sharing blocks we just created"""
-            if not name.isalnum(): raise ValueError('block hash needs to be alpha numeric')
+            def _delay_wait_for_share_block_removal(block: 'BlockHash'):
+                min_w = onionrvalues.MIN_SHARE_WAIT_DELAY_SECS
+                # Delay at least min but otherwise getBlocks timer + rand 10s
+                delay_before_remove = max(
+                                          min_w,
+                                          randbelow
+                                          (config.get
+                                           (
+                                            'timers.getBlocks',
+                                            default=10) + randbelow(11)))
+                sleep(delay_before_remove)
+                try:
+                    client_api.publicAPI.hideBlocks.remove(name)
+                except ValueError:
+                    logger.warn(
+                        f'Failed to remove {name} from waitforshare')
+
+            if not name.isalnum():
+                raise ValueError('block hash needs to be alnum')
             name = reconstructhash.reconstruct_hash(name)
             if name in client_api.publicAPI.hideBlocks:
-                client_api.publicAPI.hideBlocks.remove(name)
+                spawn(_delay_wait_for_share_block_removal)
-                return Response("removed")
+                return Response("will be removed")
             else:
                 client_api.publicAPI.hideBlocks.append(name)
                 return Response("added")

src/onionrblocks/__init__.py

@@ -1,3 +1,5 @@
 from . import insert
+from .insert import time_insert
 
 insert = insert.insert_block
+time_insert = time_insert

src/onionrblocks/insert/__init__.py

@@ -0,0 +1,4 @@
+from . import main, timeinsert
+
+insert_block = main.insert_block
+time_insert = timeinsert.time_insert

src/onionrblocks/insert/main.py

@@ -22,7 +22,7 @@ import json
 
 from onionrutils import bytesconverter, epoch
 import filepaths, onionrstorage
-from . import storagecounter
+from .. import storagecounter
 from onionrplugins import onionrevents as events
 from etc import powchoice, onionrvalues
 import config, onionrcrypto as crypto, onionrexceptions

src/onionrblocks/insert/timeinsert.py

@@ -0,0 +1,51 @@
+"""Onionr - Private P2P Communication.
+
+Wrapper to insert blocks with variable delay
+"""
+from . import main
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+
+def time_insert(*args, **kwargs):
+    """Block insert wrapper to allow for insertions independent of mixmate.
+
+    Takes exact args as insert_block, with additional keyword:
+    delay=n; where n=seconds to tell initial nodes to delay share for.
+
+    defaults to 0 or previously set value in current block meta
+    """
+    try:
+        kwargs['meta']
+    except KeyError:
+        kwargs['meta'] = {}
+
+    try:
+        delay = int(kwargs['meta']['dly'])
+    except KeyError:
+        delay = 0
+    try:
+        delay = kwargs['delay']
+        del kwargs['delay']
+    except KeyError:
+        delay = 0
+
+    # Ensure delay >=0
+    if delay < 0:
+        raise ValueError('delay cannot be less than 0')
+
+    kwargs['meta']['dly'] = delay
+
+    return main.insert_block(*args, **kwargs)

tests/test_template.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+import unittest, sys
+sys.path.append(".")
+sys.path.append("src/")
+
+
+class TestTemplate(unittest.TestCase):
+    def test_my_test(self):
+        self.assertTrue(True)
+
+unittest.main()

tests/test_timeinsert.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+import unittest, sys
+sys.path.append(".")
+sys.path.append("src/")
+
+from onionrblocks import time_insert
+from onionrblocks import onionrblockapi
+
+class TestTimeInsert(unittest.TestCase):
+    def test_time_insert_none(self):
+        bl = time_insert('test')
+        self.assertTrue(bl)
+        bl = onionrblockapi.Block(bl)
+        self.assertIs(bl.bmetadata['dly'], 0)
+
+    def test_time_insert_10(self):
+        bl = time_insert('test', delay=10)
+        self.assertTrue(bl)
+        bl = onionrblockapi.Block(bl)
+        self.assertIs(bl.bmetadata['dly'], 10)
+
+    def test_negative(self):
+        self.assertRaises(ValueError, time_insert, 'test', delay=-1)
+        self.assertRaises(ValueError, time_insert, 'test', delay=-10)
+
+
+
+unittest.main()