work on block insertion mixing

master
Kevin Froman 2019-12-27 01:53:18 -06:00
parent 87ea8d137b
commit 01f9b9b470
12 changed files with 183 additions and 40 deletions

View File

@ -5,10 +5,12 @@ Handle daemon queue commands in the communicator
import logger
from onionrplugins import onionrevents as events
from onionrutils import localcommand
from communicatorutils.uploadblocks import mixmate
from coredb import daemonqueue
import filepaths
from . import restarttor
from communicatorutils.uploadblocks import mixmate
from .. import restarttor
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -29,11 +31,12 @@ def handle_daemon_commands(comm_inst):
cmd = daemonqueue.daemon_queue()
response = ''
if cmd is not False:
events.event('daemon_command', data = {'cmd' : cmd})
events.event('daemon_command', data={'cmd': cmd})
if cmd[0] == 'shutdown':
comm_inst.shutdown = True
elif cmd[0] == 'runtimeTest':
comm_inst.shared_state.get_by_string("OnionrRunTestManager").run_tests()
comm_inst.shared_state.get_by_string(
"OnionrRunTestManager").run_tests()
elif cmd[0] == 'remove_from_insert_list':
try:
comm_inst.generating_blocks.remove(cmd[1])
@ -44,7 +47,7 @@ def handle_daemon_commands(comm_inst):
comm_inst.announce(cmd[1])
else:
logger.debug("No nodes connected. Will not introduce node.")
elif cmd[0] == 'runCheck': # deprecated
elif cmd[0] == 'runCheck': # deprecated
logger.debug('Status check; looks good.')
open(filepaths.run_check_file + '.runcheck', 'w+').close()
elif cmd[0] == 'connectedPeers':
@ -65,18 +68,21 @@ def handle_daemon_commands(comm_inst):
elif cmd[0] == 'uploadBlock':
comm_inst.blocksToUpload.append(cmd[1])
elif cmd[0] == 'uploadEvent':
localcommand.local_command('/waitforshare/' + cmd[1], post=True,
maxWait=5)
try:
mixmate.block_mixer(comm_inst.blocksToUpload, cmd[1])
except ValueError:
pass
else:
localcommand.local_command('/waitforshare/' + cmd[1], post=True, maxWait=5)
comm_inst.blocksToUpload.append(cmd[1])
else:
logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],))
logger.debug(
'Received daemon queue cmd with no handler: %s' % (cmd[0],))
if cmd[0] not in ('', None):
if response != '':
localcommand.local_command('queueResponseAdd/' + cmd[4], post=True, postData={'data': response})
localcommand.local_command('queueResponseAdd/' + cmd[4],
post=True,
postData={'data': response})
response = ''
comm_inst.decrementThreadCount('handle_daemon_commands')

View File

@ -15,7 +15,6 @@ import onionrcrypto
from communicator import onlinepeers
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by

View File

@ -38,6 +38,14 @@ def block_mixer(upload_list: List[onionrtypes.BlockHash],
to the said block list
"""
bl = onionrblockapi.Block(block_to_mix)
try:
bl.bmetadata['dly']
except (KeyError, TypeError):
pass
else:
raise ValueError
if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE:
raise ValueError

View File

@ -106,7 +106,10 @@ class BlockUploadSessionManager:
if (sess.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
sessions_to_delete.append(sess)
for sess in sessions_to_delete:
self.sessions.remove(session)
try:
self.sessions.remove(session)
except ValueError:
pass
# TODO cleanup to one round of search
# Remove the blocks from the sessions, upload list,
# and waitforshare list

View File

@ -43,6 +43,7 @@ DATABASE_LOCK_TIMEOUT = 60
# Block creation anonymization requirements
MIN_BLOCK_UPLOAD_PEER_PERCENT = 0.1
MIN_SHARE_WAIT_DELAY_SECS = 5
# Begin OnionrValues migrated values
"""Make announce take a few seconds (on average) to compute to discourage excessive node announcements"""

View File

@ -1,9 +1,33 @@
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
Misc client API endpoints too small to need their own file and that need access to the client api inst
'''
'''
Misc client API endpoints too small to need their own file
and that need access to the client api inst
"""
from typing import TYPE_CHECKING
from secrets import randbelow
import os
import subprocess
from flask import Response, Blueprint, request, send_from_directory, abort
from gevent import spawn
from gevent import sleep
import unpaddedbase32
from httpapi import apiutils
import logger
import onionrcrypto
import config
from netcontroller import NetController
from serializeddata import SerializedData
from onionrutils import mnemonickeys
from onionrutils import bytesconverter
from etc import onionrvalues
from utils import reconstructhash
from onionrcommands import restartonionr
if TYPE_CHECKING:
from onionrtypes import BlockHash
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -16,26 +40,14 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import os
import subprocess
from flask import Response, Blueprint, request, send_from_directory, abort
import unpaddedbase32
from httpapi import apiutils
import onionrcrypto, config
from netcontroller import NetController
from serializeddata import SerializedData
from onionrutils import mnemonickeys
from onionrutils import bytesconverter
from etc import onionrvalues
from utils import reconstructhash
from onionrcommands import restartonionr
"""
pub_key = onionrcrypto.pub_key.replace('=', '')
SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + f'/../../../{onionrvalues.SCRIPT_NAME}'
SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + \
f'/../../../{onionrvalues.SCRIPT_NAME}'
class PrivateEndpoints:
def __init__(self, client_api):
@ -84,12 +96,30 @@ class PrivateEndpoints:
@private_endpoints_bp.route('/waitforshare/<name>', methods=['post'])
def waitforshare(name):
'''Used to prevent the **public** api from sharing blocks we just created'''
if not name.isalnum(): raise ValueError('block hash needs to be alpha numeric')
"""Prevent the **public** api from sharing blocks we just created"""
def _delay_wait_for_share_block_removal(block: 'BlockHash'):
min_w = onionrvalues.MIN_SHARE_WAIT_DELAY_SECS
# Delay at least min but otherwise getBlocks timer + rand 10s
delay_before_remove = max(
min_w,
randbelow
(config.get
(
'timers.getBlocks',
default=10) + randbelow(11)))
sleep(delay_before_remove)
try:
client_api.publicAPI.hideBlocks.remove(name)
except ValueError:
logger.warn(
f'Failed to remove {name} from waitforshare')
if not name.isalnum():
raise ValueError('block hash needs to be alnum')
name = reconstructhash.reconstruct_hash(name)
if name in client_api.publicAPI.hideBlocks:
client_api.publicAPI.hideBlocks.remove(name)
return Response("removed")
spawn(_delay_wait_for_share_block_removal)
return Response("will be removed")
else:
client_api.publicAPI.hideBlocks.append(name)
return Response("added")

View File

@ -1,3 +1,5 @@
from . import insert
from .insert import time_insert
insert = insert.insert_block
time_insert = time_insert

View File

@ -0,0 +1,4 @@
from . import main, timeinsert
insert_block = main.insert_block
time_insert = timeinsert.time_insert

View File

@ -22,7 +22,7 @@ import json
from onionrutils import bytesconverter, epoch
import filepaths, onionrstorage
from . import storagecounter
from .. import storagecounter
from onionrplugins import onionrevents as events
from etc import powchoice, onionrvalues
import config, onionrcrypto as crypto, onionrexceptions

View File

@ -0,0 +1,51 @@
"""Onionr - Private P2P Communication.
Wrapper to insert blocks with variable delay
"""
from . import main
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
def time_insert(*args, **kwargs):
"""Block insert wrapper to allow for insertions independent of mixmate.
Takes exact args as insert_block, with additional keyword:
delay=n; where n=seconds to tell initial nodes to delay share for.
defaults to 0 or previously set value in current block meta
"""
try:
kwargs['meta']
except KeyError:
kwargs['meta'] = {}
try:
delay = int(kwargs['meta']['dly'])
except KeyError:
delay = 0
try:
delay = kwargs['delay']
del kwargs['delay']
except KeyError:
delay = 0
# Ensure delay >=0
if delay < 0:
raise ValueError('delay cannot be less than 0')
kwargs['meta']['dly'] = delay
return main.insert_block(*args, **kwargs)

11
tests/test_template.py Normal file
View File

@ -0,0 +1,11 @@
#!/usr/bin/env python3
import unittest, sys
sys.path.append(".")
sys.path.append("src/")
class TestTemplate(unittest.TestCase):
def test_my_test(self):
self.assertTrue(True)
unittest.main()

28
tests/test_timeinsert.py Normal file
View File

@ -0,0 +1,28 @@
#!/usr/bin/env python3
import unittest, sys
sys.path.append(".")
sys.path.append("src/")
from onionrblocks import time_insert
from onionrblocks import onionrblockapi
class TestTimeInsert(unittest.TestCase):
def test_time_insert_none(self):
bl = time_insert('test')
self.assertTrue(bl)
bl = onionrblockapi.Block(bl)
self.assertIs(bl.bmetadata['dly'], 0)
def test_time_insert_10(self):
bl = time_insert('test', delay=10)
self.assertTrue(bl)
bl = onionrblockapi.Block(bl)
self.assertIs(bl.bmetadata['dly'], 10)
def test_negative(self):
self.assertRaises(ValueError, time_insert, 'test', delay=-1)
self.assertRaises(ValueError, time_insert, 'test', delay=-10)
unittest.main()