dont upload blocks that arent valid (anymore?), progress removing communicator timers

master
Kevin Froman 2020-11-16 06:57:38 +00:00
parent cc703fc404
commit a775f3b26d
7 changed files with 44 additions and 25 deletions

View File

@ -16,6 +16,7 @@ conf['general']['insert_deniable_blocks'] = True
conf['general']['random_bind_ip'] = True conf['general']['random_bind_ip'] = True
conf['general']['display_header'] = True conf['general']['display_header'] = True
conf['general']['security_level'] = 0 conf['general']['security_level'] = 0
conf['general']['use_bootstrap_list'] = True
conf['onboarding']['done'] = False conf['onboarding']['done'] = False
conf['general']['minimum_block_pow'] = 5 conf['general']['minimum_block_pow'] = 5
conf['general']['minimum_send_pow'] = 5 conf['general']['minimum_send_pow'] = 5

View File

@ -26,6 +26,9 @@ conf['general']['random_bind_ip'] = False
conf['onboarding']['done'] = True conf['onboarding']['done'] = True
conf['general']['minimum_block_pow'] = block_pow conf['general']['minimum_block_pow'] = block_pow
conf['general']['minimum_send_pow'] = block_pow conf['general']['minimum_send_pow'] = block_pow
conf['general']['use_bootstrap_list'] = False
if input("Use bootstrap list? y/n").lower() == 'y':
conf['general']['use_bootstrap_list'] = True
conf['log']['file']['remove_on_exit'] = False conf['log']['file']['remove_on_exit'] = False
conf['ui']['animated_background'] = False conf['ui']['animated_background'] = False
if input('Stat reporting? y/n') == 'y': if input('Stat reporting? y/n') == 'y':

View File

@ -57,6 +57,7 @@ class OnionrCommunicatorDaemon:
# configure logger and stuff # configure logger and stuff
self.config = config self.config = config
self.shared_state = shared_state # TooManyObjects module self.shared_state = shared_state # TooManyObjects module
shared_state.add(self)
# populate kv values # populate kv values
self.kv = self.shared_state.get_by_string('DeadSimpleKV') self.kv = self.shared_state.get_by_string('DeadSimpleKV')
@ -98,7 +99,7 @@ class OnionrCommunicatorDaemon:
add_onionr_thread(onlinepeers.clear_offline_peer, [self.kv], 58) add_onionr_thread(onlinepeers.clear_offline_peer, [self.kv], 58)
add_onionr_thread( add_onionr_thread(
housekeeping.clean_old_blocks, [self.shared_state], 20, 1) housekeeping.clean_old_blocks, [self.shared_state], 10, 1)
# Discover new peers # Discover new peers
add_onionr_thread( add_onionr_thread(
@ -108,12 +109,13 @@ class OnionrCommunicatorDaemon:
# Timer for adjusting which peers # Timer for adjusting which peers
# we actively communicate to at any given time, # we actively communicate to at any given time,
# to avoid over-using peers # to avoid over-using peers
add_onionr_thread(cooldownpeer.cooldown_peer, [self.shared_state], 30, 60) add_onionr_thread(
cooldownpeer.cooldown_peer, [self.shared_state], 30, 60)
# Timer to read the upload queue and upload the entries to peers # Timer to read the upload queue and upload the entries to peers
OnionrCommunicatorTimers( add_onionr_thread(
self, uploadblocks.upload_blocks_from_communicator, uploadblocks.upload_blocks_from_communicator,
5, my_args=[self], requires_peer=True, max_threads=1) [self.shared_state], 5, 1)
# Setup direct connections # Setup direct connections
if config.get('general.ephemeral_tunnels', False): if config.get('general.ephemeral_tunnels', False):
@ -132,10 +134,7 @@ class OnionrCommunicatorDaemon:
# This timer creates deniable blocks, # This timer creates deniable blocks,
# in an attempt to further obfuscate block insertion metadata # in an attempt to further obfuscate block insertion metadata
if config.get('general.insert_deniable_blocks', True): if config.get('general.insert_deniable_blocks', True):
deniableBlockTimer = OnionrCommunicatorTimers( add_onionr_thread(deniableinserts.insert_deniable_block, [], 180, 10)
self, deniableinserts.insert_deniable_block,
180, my_args=[self], requires_peer=True, max_threads=1)
deniableBlockTimer.count = (deniableBlockTimer.frequency - 175)
if config.get('transports.tor', True): if config.get('transports.tor', True):
# Timer to check for connectivity, # Timer to check for connectivity,
@ -168,8 +167,6 @@ class OnionrCommunicatorDaemon:
# Adjust initial timer triggers # Adjust initial timer triggers
cleanupTimer.count = (cleanupTimer.frequency - 60) cleanupTimer.count = (cleanupTimer.frequency - 60)
shared_state.add(self)
if config.get('general.use_bootstrap_list', True): if config.get('general.use_bootstrap_list', True):
bootstrappeers.add_bootstrap_list_to_peer_list( bootstrappeers.add_bootstrap_list_to_peer_list(
self.kv, [], db_only=True) self.kv, [], db_only=True)

View File

@ -22,7 +22,7 @@ import onionrblocks
""" """
def insert_deniable_block(comm_inst): def insert_deniable_block():
"""Insert a fake block to make it more difficult to track real blocks.""" """Insert a fake block to make it more difficult to track real blocks."""
fakePeer = '' fakePeer = ''
chance = 10 chance = 10
@ -33,4 +33,3 @@ def insert_deniable_block(comm_inst):
onionrblocks.insert(data, header='pm', encryptType='asym', onionrblocks.insert(data, header='pm', encryptType='asym',
asymPeer=fakePeer, disableForward=True, asymPeer=fakePeer, disableForward=True,
meta={'subject': 'foo'}) meta={'subject': 'foo'})
comm_inst.decrementThreadCount('insert_deniable_block')

View File

@ -14,7 +14,9 @@ import logger
from communicatorutils import proxypicker from communicatorutils import proxypicker
import onionrexceptions import onionrexceptions
from onionrblocks import onionrblockapi as block from onionrblocks import onionrblockapi as block
from onionrblocks.blockmetadata.fromdata import get_block_metadata_from_data
from onionrutils import stringvalidators, basicrequests from onionrutils import stringvalidators, basicrequests
from onionrutils.validatemetadata import validate_metadata
from communicator import onlinepeers from communicator import onlinepeers
if TYPE_CHECKING: if TYPE_CHECKING:
from deadsimplekv import DeadSimpleKV from deadsimplekv import DeadSimpleKV
@ -35,15 +37,14 @@ if TYPE_CHECKING:
""" """
def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'): def upload_blocks_from_communicator(shared_state: 'OnionrCommunicatorDaemon'):
"""Accept a communicator instance + upload blocks from its upload queue.""" """Accept a communicator instance + upload blocks from its upload queue."""
"""when inserting a block, we try to upload """when inserting a block, we try to upload
it to a few peers to add some deniability & increase functionality""" it to a few peers to add some deniability & increase functionality"""
kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV") kv: "DeadSimpleKV" = shared_state.get_by_string("DeadSimpleKV")
TIMER_NAME = "upload_blocks_from_communicator"
session_manager: sessionmanager.BlockUploadSessionManager session_manager: sessionmanager.BlockUploadSessionManager
session_manager = comm_inst.shared_state.get( session_manager = shared_state.get(
sessionmanager.BlockUploadSessionManager) sessionmanager.BlockUploadSessionManager)
tried_peers: UserID = [] tried_peers: UserID = []
finishedUploads = [] finishedUploads = []
@ -53,7 +54,7 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
def remove_from_hidden(bl): def remove_from_hidden(bl):
sleep(60) sleep(60)
try: try:
comm_inst.shared_state.get_by_string( shared_state.get_by_string(
'PublicAPI').hideBlocks.remove(bl) 'PublicAPI').hideBlocks.remove(bl)
except ValueError: except ValueError:
pass pass
@ -62,7 +63,6 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
for bl in kv.get('blocksToUpload'): for bl in kv.get('blocksToUpload'):
if not stringvalidators.validate_hash(bl): if not stringvalidators.validate_hash(bl):
logger.warn('Requested to upload invalid block', terminal=True) logger.warn('Requested to upload invalid block', terminal=True)
comm_inst.decrementThreadCount(TIMER_NAME)
return return
session = session_manager.add_session(bl) session = session_manager.add_session(bl)
for _ in range(min(len(kv.get('onlinePeers')), 6)): for _ in range(min(len(kv.get('onlinePeers')), 6)):
@ -92,10 +92,22 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
data = block.Block(bl).getRaw() data = block.Block(bl).getRaw()
if not data: if not data:
logger.warn( logger.warn(
f"Couldn't data for block in upload list {bl}", f"Couldn't get data for block in upload list {bl}",
terminal=True) terminal=True)
raise onionrexceptions.NoDataAvailable raise onionrexceptions.NoDataAvailable
except onionrexceptions.NoDataAvailable: try:
def __check_metadata():
metadata = get_block_metadata_from_data(data)[0]
if not validate_metadata(metadata, data):
logger.warn(
f"Metadata for uploading block not valid {bl}")
raise onionrexceptions.InvalidMetadata
__check_metadata()
except onionrexceptions.DataExists:
pass
except( # noqa
onionrexceptions.NoDataAvailable,
onionrexceptions.InvalidMetadata) as _:
finishedUploads.append(bl) finishedUploads.append(bl)
break break
proxy_type = proxypicker.pick_proxy(peer) proxy_type = proxypicker.pick_proxy(peer)
@ -116,7 +128,9 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
else: else:
session.fail() session.fail()
session.fail_peer(peer) session.fail_peer(peer)
comm_inst.getPeerProfileInstance(peer).addScore(-5) shared_state.get_by_string(
'OnionrCommunicatorDaemon').getPeerProfileInstance(
peer).addScore(-5)
logger.warn( logger.warn(
f'Failed to upload {bl[:8]}, reason: {resp}', f'Failed to upload {bl[:8]}, reason: {resp}',
terminal=True) terminal=True)
@ -127,9 +141,8 @@ def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
try: try:
kv.get('blocksToUpload').remove(x) kv.get('blocksToUpload').remove(x)
comm_inst.shared_state.get_by_string( shared_state.get_by_string(
'PublicAPI').hideBlocks.remove(x) 'PublicAPI').hideBlocks.remove(x)
except ValueError: except ValueError:
pass pass
comm_inst.decrementThreadCount(TIMER_NAME)

View File

@ -116,6 +116,8 @@ class PrivateEndpoints:
SerializedData).get_stats()) SerializedData).get_stats())
except AttributeError: except AttributeError:
pass pass
except FileNotFoundError:
pass
@private_endpoints_bp.route('/getuptime') @private_endpoints_bp.route('/getuptime')
def show_uptime(): def show_uptime():

View File

@ -71,6 +71,10 @@ def accept_upload(request):
except onionrexceptions.PlaintextNotSupported: except onionrexceptions.PlaintextNotSupported:
logger.debug("attempted plaintext upload to us: {b_hash}") logger.debug("attempted plaintext upload to us: {b_hash}")
resp = 'failure' resp = 'failure'
except onionrexceptions.InvalidMetadata:
logger.debug(
'uploaded block {b_hash} has invalid metadata')
resp = 'failure'
if resp == 'failure': if resp == 'failure':
abort(400) abort(400)
elif resp == 'size': elif resp == 'size':