From 01f9b9b470939e51d45340bc56c134bf6c2a5026 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 27 Dec 2019 01:53:18 -0600 Subject: [PATCH 1/3] work on block insertion mixing --- .../__init__.py} | 26 +++--- .../uploadblocks/__init__.py | 1 - .../uploadblocks/mixmate/__init__.py | 8 ++ .../uploadblocks/sessionmanager.py | 5 +- src/etc/onionrvalues.py | 1 + src/httpapi/miscclientapi/endpoints.py | 82 +++++++++++++------ src/onionrblocks/__init__.py | 4 +- src/onionrblocks/insert/__init__.py | 4 + .../{insert.py => insert/main.py} | 2 +- src/onionrblocks/insert/timeinsert.py | 51 ++++++++++++ tests/test_template.py | 11 +++ tests/test_timeinsert.py | 28 +++++++ 12 files changed, 183 insertions(+), 40 deletions(-) rename src/communicatorutils/{daemonqueuehandler.py => daemonqueuehandler/__init__.py} (81%) create mode 100644 src/onionrblocks/insert/__init__.py rename src/onionrblocks/{insert.py => insert/main.py} (99%) create mode 100644 src/onionrblocks/insert/timeinsert.py create mode 100644 tests/test_template.py create mode 100644 tests/test_timeinsert.py diff --git a/src/communicatorutils/daemonqueuehandler.py b/src/communicatorutils/daemonqueuehandler/__init__.py similarity index 81% rename from src/communicatorutils/daemonqueuehandler.py rename to src/communicatorutils/daemonqueuehandler/__init__.py index 9cddc793..01a8a606 100755 --- a/src/communicatorutils/daemonqueuehandler.py +++ b/src/communicatorutils/daemonqueuehandler/__init__.py @@ -5,10 +5,12 @@ Handle daemon queue commands in the communicator import logger from onionrplugins import onionrevents as events from onionrutils import localcommand +from communicatorutils.uploadblocks import mixmate from coredb import daemonqueue import filepaths -from . import restarttor -from communicatorutils.uploadblocks import mixmate + +from .. import restarttor + """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -29,11 +31,12 @@ def handle_daemon_commands(comm_inst): cmd = daemonqueue.daemon_queue() response = '' if cmd is not False: - events.event('daemon_command', data = {'cmd' : cmd}) + events.event('daemon_command', data={'cmd': cmd}) if cmd[0] == 'shutdown': comm_inst.shutdown = True elif cmd[0] == 'runtimeTest': - comm_inst.shared_state.get_by_string("OnionrRunTestManager").run_tests() + comm_inst.shared_state.get_by_string( + "OnionrRunTestManager").run_tests() elif cmd[0] == 'remove_from_insert_list': try: comm_inst.generating_blocks.remove(cmd[1]) @@ -44,7 +47,7 @@ def handle_daemon_commands(comm_inst): comm_inst.announce(cmd[1]) else: logger.debug("No nodes connected. Will not introduce node.") - elif cmd[0] == 'runCheck': # deprecated + elif cmd[0] == 'runCheck': # deprecated logger.debug('Status check; looks good.') open(filepaths.run_check_file + '.runcheck', 'w+').close() elif cmd[0] == 'connectedPeers': @@ -65,18 +68,21 @@ def handle_daemon_commands(comm_inst): elif cmd[0] == 'uploadBlock': comm_inst.blocksToUpload.append(cmd[1]) elif cmd[0] == 'uploadEvent': + localcommand.local_command('/waitforshare/' + cmd[1], post=True, + maxWait=5) try: mixmate.block_mixer(comm_inst.blocksToUpload, cmd[1]) except ValueError: - pass - else: - localcommand.local_command('/waitforshare/' + cmd[1], post=True, maxWait=5) + comm_inst.blocksToUpload.append(cmd[1]) else: - logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],)) + logger.debug( + 'Received daemon queue cmd with no handler: %s' % (cmd[0],)) if cmd[0] not in ('', None): if response != '': - localcommand.local_command('queueResponseAdd/' + cmd[4], post=True, postData={'data': response}) + localcommand.local_command('queueResponseAdd/' + cmd[4], + post=True, + postData={'data': response}) response = '' comm_inst.decrementThreadCount('handle_daemon_commands') diff --git a/src/communicatorutils/uploadblocks/__init__.py b/src/communicatorutils/uploadblocks/__init__.py index d0a53108..35493b64 100755 --- a/src/communicatorutils/uploadblocks/__init__.py +++ b/src/communicatorutils/uploadblocks/__init__.py @@ -15,7 +15,6 @@ import onionrcrypto from communicator import onlinepeers if TYPE_CHECKING: from communicator import OnionrCommunicatorDaemon - """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/src/communicatorutils/uploadblocks/mixmate/__init__.py b/src/communicatorutils/uploadblocks/mixmate/__init__.py index ce50be7e..8c91929c 100644 --- a/src/communicatorutils/uploadblocks/mixmate/__init__.py +++ b/src/communicatorutils/uploadblocks/mixmate/__init__.py @@ -38,6 +38,14 @@ def block_mixer(upload_list: List[onionrtypes.BlockHash], to the said block list """ bl = onionrblockapi.Block(block_to_mix) + + try: + bl.bmetadata['dly'] + except (KeyError, TypeError): + pass + else: + raise ValueError + if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE: raise ValueError diff --git a/src/communicatorutils/uploadblocks/sessionmanager.py b/src/communicatorutils/uploadblocks/sessionmanager.py index 81faa668..43ee5279 100644 --- a/src/communicatorutils/uploadblocks/sessionmanager.py +++ b/src/communicatorutils/uploadblocks/sessionmanager.py @@ -106,7 +106,10 @@ class BlockUploadSessionManager: if (sess.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT: sessions_to_delete.append(sess) for sess in sessions_to_delete: - self.sessions.remove(session) + try: + self.sessions.remove(session) + except ValueError: + pass # TODO cleanup to one round of search # Remove the blocks from the sessions, upload list, # and waitforshare list diff --git a/src/etc/onionrvalues.py b/src/etc/onionrvalues.py index c4c30b65..7f2c975f 100755 --- a/src/etc/onionrvalues.py +++ b/src/etc/onionrvalues.py @@ -43,6 +43,7 @@ DATABASE_LOCK_TIMEOUT = 60 # Block creation anonymization requirements MIN_BLOCK_UPLOAD_PEER_PERCENT = 0.1 +MIN_SHARE_WAIT_DELAY_SECS = 5 # Begin OnionrValues migrated values """Make announce take a few seconds (on average) to compute to discourage excessive node announcements""" diff --git a/src/httpapi/miscclientapi/endpoints.py b/src/httpapi/miscclientapi/endpoints.py index e92bfe05..a0fa5d72 100644 --- a/src/httpapi/miscclientapi/endpoints.py +++ b/src/httpapi/miscclientapi/endpoints.py @@ -1,9 +1,33 @@ -''' - Onionr - Private P2P Communication +"""Onionr - Private P2P Communication. - Misc client API endpoints too small to need their own file and that need access to the client api inst -''' -''' +Misc client API endpoints too small to need their own file +and that need access to the client api inst +""" +from typing import TYPE_CHECKING +from secrets import randbelow + +import os +import subprocess + +from flask import Response, Blueprint, request, send_from_directory, abort +from gevent import spawn +from gevent import sleep +import unpaddedbase32 + +from httpapi import apiutils +import logger +import onionrcrypto +import config +from netcontroller import NetController +from serializeddata import SerializedData +from onionrutils import mnemonickeys +from onionrutils import bytesconverter +from etc import onionrvalues +from utils import reconstructhash +from onionrcommands import restartonionr +if TYPE_CHECKING: + from onionrtypes import BlockHash +""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -16,26 +40,14 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' -import os -import subprocess - -from flask import Response, Blueprint, request, send_from_directory, abort -import unpaddedbase32 - -from httpapi import apiutils -import onionrcrypto, config -from netcontroller import NetController -from serializeddata import SerializedData -from onionrutils import mnemonickeys -from onionrutils import bytesconverter -from etc import onionrvalues -from utils import reconstructhash -from onionrcommands import restartonionr +""" pub_key = onionrcrypto.pub_key.replace('=', '') -SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + f'/../../../{onionrvalues.SCRIPT_NAME}' +SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + \ + f'/../../../{onionrvalues.SCRIPT_NAME}' + + class PrivateEndpoints: def __init__(self, client_api): @@ -84,12 +96,30 @@ class PrivateEndpoints: @private_endpoints_bp.route('/waitforshare/', methods=['post']) def waitforshare(name): - '''Used to prevent the **public** api from sharing blocks we just created''' - if not name.isalnum(): raise ValueError('block hash needs to be alpha numeric') + """Prevent the **public** api from sharing blocks we just created""" + def _delay_wait_for_share_block_removal(block: 'BlockHash'): + min_w = onionrvalues.MIN_SHARE_WAIT_DELAY_SECS + # Delay at least min but otherwise getBlocks timer + rand 10s + delay_before_remove = max( + min_w, + randbelow + (config.get + ( + 'timers.getBlocks', + default=10) + randbelow(11))) + sleep(delay_before_remove) + try: + client_api.publicAPI.hideBlocks.remove(name) + except ValueError: + logger.warn( + f'Failed to remove {name} from waitforshare') + + if not name.isalnum(): + raise ValueError('block hash needs to be alnum') name = reconstructhash.reconstruct_hash(name) if name in client_api.publicAPI.hideBlocks: - client_api.publicAPI.hideBlocks.remove(name) - return Response("removed") + spawn(_delay_wait_for_share_block_removal) + return Response("will be removed") else: client_api.publicAPI.hideBlocks.append(name) return Response("added") diff --git a/src/onionrblocks/__init__.py b/src/onionrblocks/__init__.py index 93e9fd3f..91a6b1a1 100644 --- a/src/onionrblocks/__init__.py +++ b/src/onionrblocks/__init__.py @@ -1,3 +1,5 @@ from . import insert +from .insert import time_insert -insert = insert.insert_block \ No newline at end of file +insert = insert.insert_block +time_insert = time_insert \ No newline at end of file diff --git a/src/onionrblocks/insert/__init__.py b/src/onionrblocks/insert/__init__.py new file mode 100644 index 00000000..e516f6b6 --- /dev/null +++ b/src/onionrblocks/insert/__init__.py @@ -0,0 +1,4 @@ +from . import main, timeinsert + +insert_block = main.insert_block +time_insert = timeinsert.time_insert \ No newline at end of file diff --git a/src/onionrblocks/insert.py b/src/onionrblocks/insert/main.py similarity index 99% rename from src/onionrblocks/insert.py rename to src/onionrblocks/insert/main.py index 29300b03..88a6c8fe 100644 --- a/src/onionrblocks/insert.py +++ b/src/onionrblocks/insert/main.py @@ -22,7 +22,7 @@ import json from onionrutils import bytesconverter, epoch import filepaths, onionrstorage -from . import storagecounter +from .. import storagecounter from onionrplugins import onionrevents as events from etc import powchoice, onionrvalues import config, onionrcrypto as crypto, onionrexceptions diff --git a/src/onionrblocks/insert/timeinsert.py b/src/onionrblocks/insert/timeinsert.py new file mode 100644 index 00000000..41d8fcaa --- /dev/null +++ b/src/onionrblocks/insert/timeinsert.py @@ -0,0 +1,51 @@ +"""Onionr - Private P2P Communication. + +Wrapper to insert blocks with variable delay +""" +from . import main +""" + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + + +def time_insert(*args, **kwargs): + """Block insert wrapper to allow for insertions independent of mixmate. + + Takes exact args as insert_block, with additional keyword: + delay=n; where n=seconds to tell initial nodes to delay share for. + + defaults to 0 or previously set value in current block meta + """ + try: + kwargs['meta'] + except KeyError: + kwargs['meta'] = {} + + try: + delay = int(kwargs['meta']['dly']) + except KeyError: + delay = 0 + try: + delay = kwargs['delay'] + del kwargs['delay'] + except KeyError: + delay = 0 + + # Ensure delay >=0 + if delay < 0: + raise ValueError('delay cannot be less than 0') + + kwargs['meta']['dly'] = delay + + return main.insert_block(*args, **kwargs) diff --git a/tests/test_template.py b/tests/test_template.py new file mode 100644 index 00000000..406b09de --- /dev/null +++ b/tests/test_template.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +import unittest, sys +sys.path.append(".") +sys.path.append("src/") + + +class TestTemplate(unittest.TestCase): + def test_my_test(self): + self.assertTrue(True) + +unittest.main() diff --git a/tests/test_timeinsert.py b/tests/test_timeinsert.py new file mode 100644 index 00000000..3d637020 --- /dev/null +++ b/tests/test_timeinsert.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import unittest, sys +sys.path.append(".") +sys.path.append("src/") + +from onionrblocks import time_insert +from onionrblocks import onionrblockapi + +class TestTimeInsert(unittest.TestCase): + def test_time_insert_none(self): + bl = time_insert('test') + self.assertTrue(bl) + bl = onionrblockapi.Block(bl) + self.assertIs(bl.bmetadata['dly'], 0) + + def test_time_insert_10(self): + bl = time_insert('test', delay=10) + self.assertTrue(bl) + bl = onionrblockapi.Block(bl) + self.assertIs(bl.bmetadata['dly'], 10) + + def test_negative(self): + self.assertRaises(ValueError, time_insert, 'test', delay=-1) + self.assertRaises(ValueError, time_insert, 'test', delay=-10) + + + +unittest.main() From 66d24e35357cf4a99974176117a4e37bf9554323 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 22 Jan 2020 10:30:15 +0000 Subject: [PATCH 2/3] Bump urllib3 from 1.25.7 to 1.25.8 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.25.7 to 1.25.8. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/master/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.25.7...1.25.8) Signed-off-by: dependabot-preview[bot] --- requirements.in | 2 +- requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.in b/requirements.in index 2c886a9e..128725b4 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,4 @@ -urllib3==1.25.7 +urllib3==1.25.8 requests==2.22.0 PyNaCl==1.3.0 gevent==1.4.0 diff --git a/requirements.txt b/requirements.txt index ad8f91f1..5dabaa4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -195,9 +195,9 @@ toomanyobjs==1.1.0 \ unpaddedbase32==0.2.0 \ --hash=sha256:4aacee75f8fd6c8cf129842ecba45ca59c11bfb13dae19d86f32b48fa3715403 \ --hash=sha256:b7b780c31d27d55e66abf6c221216a35690ee8892c2daacff7f2528e229bd9c3 -urllib3==1.25.7 \ - --hash=sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293 \ - --hash=sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745 +urllib3==1.25.8 \ + --hash=sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc \ + --hash=sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc werkzeug==0.15.5 \ --hash=sha256:87ae4e5b5366da2347eb3116c0e6c681a0e939a33b2805e2c0cbd282664932c4 \ --hash=sha256:a13b74dd3c45f758d4ebdb224be8f1ab8ef58b3c0ffc1783a8c7d9f4f50227e6 \ From 73993a67e5b9bf319312fdb81d8a57cbbf1d8ab6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2020 23:37:32 +0000 Subject: [PATCH 3/3] Bump jinja2 from 2.10.3 to 2.11.0 Bumps [jinja2](https://github.com/pallets/jinja) from 2.10.3 to 2.11.0. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/master/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/2.10.3...2.11.0) Signed-off-by: dependabot-preview[bot] --- requirements.in | 2 +- requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.in b/requirements.in index 2c886a9e..b7f09357 100644 --- a/requirements.in +++ b/requirements.in @@ -8,7 +8,7 @@ stem==1.8.0 deadsimplekv==0.2.0 unpaddedbase32==0.2.0 streamedrequests==1.0.0 -jinja2==2.10.3 +jinja2==2.11.0 toomanyobjs==1.1.0 niceware==0.2.1 psutil==5.6.7 diff --git a/requirements.txt b/requirements.txt index ad8f91f1..79ce6c90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -104,9 +104,9 @@ itsdangerous==1.1.0 \ --hash=sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19 \ --hash=sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749 \ # via flask -jinja2==2.10.3 \ - --hash=sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f \ - --hash=sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de +jinja2==2.11.0 \ + --hash=sha256:6e7a3c2934694d59ad334c93dd1b6c96699cf24c53fdb8ec848ac6b23e685734 \ + --hash=sha256:d6609ae5ec3d56212ca7d802eda654eaf2310000816ce815361041465b108be4 markupsafe==1.1.1 \ --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \ --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \