From a775f3b26d7f3177826b5bcbc296769a34c77daf Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 16 Nov 2020 06:57:38 +0000 Subject: [PATCH] dont upload blocks that arent valid (anymore?), progress removing communicator timers --- scripts/disable-dev-config.py | 1 + scripts/enable-dev-config.py | 3 ++ src/communicator/__init__.py | 19 ++++------ src/communicatorutils/deniableinserts.py | 3 +- .../uploadblocks/__init__.py | 37 +++++++++++++------ src/httpapi/miscclientapi/endpoints.py | 2 + src/httpapi/miscpublicapi/upload.py | 4 ++ 7 files changed, 44 insertions(+), 25 deletions(-) diff --git a/scripts/disable-dev-config.py b/scripts/disable-dev-config.py index b1dced09..ca337a12 100755 --- a/scripts/disable-dev-config.py +++ b/scripts/disable-dev-config.py @@ -16,6 +16,7 @@ conf['general']['insert_deniable_blocks'] = True conf['general']['random_bind_ip'] = True conf['general']['display_header'] = True conf['general']['security_level'] = 0 +conf['general']['use_bootstrap_list'] = True conf['onboarding']['done'] = False conf['general']['minimum_block_pow'] = 5 conf['general']['minimum_send_pow'] = 5 diff --git a/scripts/enable-dev-config.py b/scripts/enable-dev-config.py index 152f1b43..b4be790d 100755 --- a/scripts/enable-dev-config.py +++ b/scripts/enable-dev-config.py @@ -26,6 +26,9 @@ conf['general']['random_bind_ip'] = False conf['onboarding']['done'] = True conf['general']['minimum_block_pow'] = block_pow conf['general']['minimum_send_pow'] = block_pow +conf['general']['use_bootstrap_list'] = False +if input("Use bootstrap list? y/n").lower() == 'y': + conf['general']['use_bootstrap_list'] = True conf['log']['file']['remove_on_exit'] = False conf['ui']['animated_background'] = False if input('Stat reporting? y/n') == 'y': diff --git a/src/communicator/__init__.py b/src/communicator/__init__.py index db79a6b2..aa91747f 100755 --- a/src/communicator/__init__.py +++ b/src/communicator/__init__.py @@ -57,6 +57,7 @@ class OnionrCommunicatorDaemon: # configure logger and stuff self.config = config self.shared_state = shared_state # TooManyObjects module + shared_state.add(self) # populate kv values self.kv = self.shared_state.get_by_string('DeadSimpleKV') @@ -98,7 +99,7 @@ class OnionrCommunicatorDaemon: add_onionr_thread(onlinepeers.clear_offline_peer, [self.kv], 58) add_onionr_thread( - housekeeping.clean_old_blocks, [self.shared_state], 20, 1) + housekeeping.clean_old_blocks, [self.shared_state], 10, 1) # Discover new peers add_onionr_thread( @@ -108,12 +109,13 @@ class OnionrCommunicatorDaemon: # Timer for adjusting which peers # we actively communicate to at any given time, # to avoid over-using peers - add_onionr_thread(cooldownpeer.cooldown_peer, [self.shared_state], 30, 60) + add_onionr_thread( + cooldownpeer.cooldown_peer, [self.shared_state], 30, 60) # Timer to read the upload queue and upload the entries to peers - OnionrCommunicatorTimers( - self, uploadblocks.upload_blocks_from_communicator, - 5, my_args=[self], requires_peer=True, max_threads=1) + add_onionr_thread( + uploadblocks.upload_blocks_from_communicator, + [self.shared_state], 5, 1) # Setup direct connections if config.get('general.ephemeral_tunnels', False): @@ -132,10 +134,7 @@ class OnionrCommunicatorDaemon: # This timer creates deniable blocks, # in an attempt to further obfuscate block insertion metadata if config.get('general.insert_deniable_blocks', True): - deniableBlockTimer = OnionrCommunicatorTimers( - self, deniableinserts.insert_deniable_block, - 180, my_args=[self], requires_peer=True, max_threads=1) - deniableBlockTimer.count = (deniableBlockTimer.frequency - 175) + add_onionr_thread(deniableinserts.insert_deniable_block, [], 180, 10) if config.get('transports.tor', True): # Timer to check for connectivity, @@ -168,8 +167,6 @@ class OnionrCommunicatorDaemon: # Adjust initial timer triggers cleanupTimer.count = (cleanupTimer.frequency - 60) - shared_state.add(self) - if config.get('general.use_bootstrap_list', True): bootstrappeers.add_bootstrap_list_to_peer_list( self.kv, [], db_only=True) diff --git a/src/communicatorutils/deniableinserts.py b/src/communicatorutils/deniableinserts.py index 3252d3a3..26c34163 100755 --- a/src/communicatorutils/deniableinserts.py +++ b/src/communicatorutils/deniableinserts.py @@ -22,7 +22,7 @@ import onionrblocks """ -def insert_deniable_block(comm_inst): +def insert_deniable_block(): """Insert a fake block to make it more difficult to track real blocks.""" fakePeer = '' chance = 10 @@ -33,4 +33,3 @@ def insert_deniable_block(comm_inst): onionrblocks.insert(data, header='pm', encryptType='asym', asymPeer=fakePeer, disableForward=True, meta={'subject': 'foo'}) - comm_inst.decrementThreadCount('insert_deniable_block') diff --git a/src/communicatorutils/uploadblocks/__init__.py b/src/communicatorutils/uploadblocks/__init__.py index 265f678f..8bf23be6 100755 --- a/src/communicatorutils/uploadblocks/__init__.py +++ b/src/communicatorutils/uploadblocks/__init__.py @@ -14,7 +14,9 @@ import logger from communicatorutils import proxypicker import onionrexceptions from onionrblocks import onionrblockapi as block +from onionrblocks.blockmetadata.fromdata import get_block_metadata_from_data from onionrutils import stringvalidators, basicrequests +from onionrutils.validatemetadata import validate_metadata from communicator import onlinepeers if TYPE_CHECKING: from deadsimplekv import DeadSimpleKV @@ -35,15 +37,14 @@ if TYPE_CHECKING: """ -def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'): +def upload_blocks_from_communicator(shared_state: 'OnionrCommunicatorDaemon'): """Accept a communicator instance + upload blocks from its upload queue.""" """when inserting a block, we try to upload it to a few peers to add some deniability & increase functionality""" - kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV") - TIMER_NAME = "upload_blocks_from_communicator" + kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV") session_manager: sessionmanager.BlockUploadSessionManager - session_manager = comm_inst.shared_state.get( + session_manager = shared_state.get( sessionmanager.BlockUploadSessionManager) tried_peers: UserID = [] finishedUploads = [] @@ -53,7 +54,7 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'): def remove_from_hidden(bl): sleep(60) try: - comm_inst.shared_state.get_by_string( + shared_state.get_by_string( 'PublicAPI').hideBlocks.remove(bl) except ValueError: pass @@ -62,7 +63,6 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'): for bl in kv.get('blocksToUpload'): if not stringvalidators.validate_hash(bl): logger.warn('Requested to upload invalid block', terminal=True) - comm_inst.decrementThreadCount(TIMER_NAME) return session = session_manager.add_session(bl) for _ in range(min(len(kv.get('onlinePeers')), 6)): @@ -92,10 +92,22 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'): data = block.Block(bl).getRaw() if not data: logger.warn( - f"Couldn't data for block in upload list {bl}", + f"Couldn't get data for block in upload list {bl}", terminal=True) raise onionrexceptions.NoDataAvailable - except onionrexceptions.NoDataAvailable: + try: + def __check_metadata(): + metadata = get_block_metadata_from_data(data)[0] + if not validate_metadata(metadata, data): + logger.warn( + f"Metadata for uploading block not valid {bl}") + raise onionrexceptions.InvalidMetadata + __check_metadata() + except onionrexceptions.DataExists: + pass + except( # noqa + onionrexceptions.NoDataAvailable, + onionrexceptions.InvalidMetadata) as _: finishedUploads.append(bl) break proxy_type = proxypicker.pick_proxy(peer) @@ -116,7 +128,9 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'): else: session.fail() session.fail_peer(peer) - comm_inst.getPeerProfileInstance(peer).addScore(-5) + shared_state.get_by_string( + 'OnionrCommunicatorDaemon').getPeerProfileInstance( + peer).addScore(-5) logger.warn( f'Failed to upload {bl[:8]}, reason: {resp}', terminal=True) @@ -127,9 +141,8 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'): try: kv.get('blocksToUpload').remove(x) - comm_inst.shared_state.get_by_string( + shared_state.get_by_string( 'PublicAPI').hideBlocks.remove(x) except ValueError: - pass - comm_inst.decrementThreadCount(TIMER_NAME) + pass \ No newline at end of file diff --git a/src/httpapi/miscclientapi/endpoints.py b/src/httpapi/miscclientapi/endpoints.py index 0eec7d6b..86270865 100644 --- a/src/httpapi/miscclientapi/endpoints.py +++ b/src/httpapi/miscclientapi/endpoints.py @@ -116,6 +116,8 @@ class PrivateEndpoints: SerializedData).get_stats()) except AttributeError: pass + except FileNotFoundError: + pass @private_endpoints_bp.route('/getuptime') def show_uptime(): diff --git a/src/httpapi/miscpublicapi/upload.py b/src/httpapi/miscpublicapi/upload.py index e165581b..de3f7243 100755 --- a/src/httpapi/miscpublicapi/upload.py +++ b/src/httpapi/miscpublicapi/upload.py @@ -71,6 +71,10 @@ def accept_upload(request): except onionrexceptions.PlaintextNotSupported: logger.debug("attempted plaintext upload to us: {b_hash}") resp = 'failure' + except onionrexceptions.InvalidMetadata: + logger.debug( + 'uploaded block {b_hash} has invalid metadata') + resp = 'failure' if resp == 'failure': abort(400) elif resp == 'size':