work on block insertion mixing
parent
87ea8d137b
commit
01f9b9b470
|
@ -5,10 +5,12 @@ Handle daemon queue commands in the communicator
|
|||
import logger
|
||||
from onionrplugins import onionrevents as events
|
||||
from onionrutils import localcommand
|
||||
from communicatorutils.uploadblocks import mixmate
|
||||
from coredb import daemonqueue
|
||||
import filepaths
|
||||
from . import restarttor
|
||||
from communicatorutils.uploadblocks import mixmate
|
||||
|
||||
from .. import restarttor
|
||||
|
||||
"""
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
|
@ -29,11 +31,12 @@ def handle_daemon_commands(comm_inst):
|
|||
cmd = daemonqueue.daemon_queue()
|
||||
response = ''
|
||||
if cmd is not False:
|
||||
events.event('daemon_command', data = {'cmd' : cmd})
|
||||
events.event('daemon_command', data={'cmd': cmd})
|
||||
if cmd[0] == 'shutdown':
|
||||
comm_inst.shutdown = True
|
||||
elif cmd[0] == 'runtimeTest':
|
||||
comm_inst.shared_state.get_by_string("OnionrRunTestManager").run_tests()
|
||||
comm_inst.shared_state.get_by_string(
|
||||
"OnionrRunTestManager").run_tests()
|
||||
elif cmd[0] == 'remove_from_insert_list':
|
||||
try:
|
||||
comm_inst.generating_blocks.remove(cmd[1])
|
||||
|
@ -44,7 +47,7 @@ def handle_daemon_commands(comm_inst):
|
|||
comm_inst.announce(cmd[1])
|
||||
else:
|
||||
logger.debug("No nodes connected. Will not introduce node.")
|
||||
elif cmd[0] == 'runCheck': # deprecated
|
||||
elif cmd[0] == 'runCheck': # deprecated
|
||||
logger.debug('Status check; looks good.')
|
||||
open(filepaths.run_check_file + '.runcheck', 'w+').close()
|
||||
elif cmd[0] == 'connectedPeers':
|
||||
|
@ -65,18 +68,21 @@ def handle_daemon_commands(comm_inst):
|
|||
elif cmd[0] == 'uploadBlock':
|
||||
comm_inst.blocksToUpload.append(cmd[1])
|
||||
elif cmd[0] == 'uploadEvent':
|
||||
localcommand.local_command('/waitforshare/' + cmd[1], post=True,
|
||||
maxWait=5)
|
||||
try:
|
||||
mixmate.block_mixer(comm_inst.blocksToUpload, cmd[1])
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
localcommand.local_command('/waitforshare/' + cmd[1], post=True, maxWait=5)
|
||||
comm_inst.blocksToUpload.append(cmd[1])
|
||||
else:
|
||||
logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],))
|
||||
logger.debug(
|
||||
'Received daemon queue cmd with no handler: %s' % (cmd[0],))
|
||||
|
||||
if cmd[0] not in ('', None):
|
||||
if response != '':
|
||||
localcommand.local_command('queueResponseAdd/' + cmd[4], post=True, postData={'data': response})
|
||||
localcommand.local_command('queueResponseAdd/' + cmd[4],
|
||||
post=True,
|
||||
postData={'data': response})
|
||||
response = ''
|
||||
|
||||
comm_inst.decrementThreadCount('handle_daemon_commands')
|
|
@ -15,7 +15,6 @@ import onionrcrypto
|
|||
from communicator import onlinepeers
|
||||
if TYPE_CHECKING:
|
||||
from communicator import OnionrCommunicatorDaemon
|
||||
|
||||
"""
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
|
|
|
@ -38,6 +38,14 @@ def block_mixer(upload_list: List[onionrtypes.BlockHash],
|
|||
to the said block list
|
||||
"""
|
||||
bl = onionrblockapi.Block(block_to_mix)
|
||||
|
||||
try:
|
||||
bl.bmetadata['dly']
|
||||
except (KeyError, TypeError):
|
||||
pass
|
||||
else:
|
||||
raise ValueError
|
||||
|
||||
if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE:
|
||||
raise ValueError
|
||||
|
||||
|
|
|
@ -106,7 +106,10 @@ class BlockUploadSessionManager:
|
|||
if (sess.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
|
||||
sessions_to_delete.append(sess)
|
||||
for sess in sessions_to_delete:
|
||||
self.sessions.remove(session)
|
||||
try:
|
||||
self.sessions.remove(session)
|
||||
except ValueError:
|
||||
pass
|
||||
# TODO cleanup to one round of search
|
||||
# Remove the blocks from the sessions, upload list,
|
||||
# and waitforshare list
|
||||
|
|
|
@ -43,6 +43,7 @@ DATABASE_LOCK_TIMEOUT = 60
|
|||
|
||||
# Block creation anonymization requirements
|
||||
MIN_BLOCK_UPLOAD_PEER_PERCENT = 0.1
|
||||
MIN_SHARE_WAIT_DELAY_SECS = 5
|
||||
|
||||
# Begin OnionrValues migrated values
|
||||
"""Make announce take a few seconds (on average) to compute to discourage excessive node announcements"""
|
||||
|
|
|
@ -1,9 +1,33 @@
|
|||
'''
|
||||
Onionr - Private P2P Communication
|
||||
"""Onionr - Private P2P Communication.
|
||||
|
||||
Misc client API endpoints too small to need their own file and that need access to the client api inst
|
||||
'''
|
||||
'''
|
||||
Misc client API endpoints too small to need their own file
|
||||
and that need access to the client api inst
|
||||
"""
|
||||
from typing import TYPE_CHECKING
|
||||
from secrets import randbelow
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from flask import Response, Blueprint, request, send_from_directory, abort
|
||||
from gevent import spawn
|
||||
from gevent import sleep
|
||||
import unpaddedbase32
|
||||
|
||||
from httpapi import apiutils
|
||||
import logger
|
||||
import onionrcrypto
|
||||
import config
|
||||
from netcontroller import NetController
|
||||
from serializeddata import SerializedData
|
||||
from onionrutils import mnemonickeys
|
||||
from onionrutils import bytesconverter
|
||||
from etc import onionrvalues
|
||||
from utils import reconstructhash
|
||||
from onionrcommands import restartonionr
|
||||
if TYPE_CHECKING:
|
||||
from onionrtypes import BlockHash
|
||||
"""
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
|
@ -16,26 +40,14 @@
|
|||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from flask import Response, Blueprint, request, send_from_directory, abort
|
||||
import unpaddedbase32
|
||||
|
||||
from httpapi import apiutils
|
||||
import onionrcrypto, config
|
||||
from netcontroller import NetController
|
||||
from serializeddata import SerializedData
|
||||
from onionrutils import mnemonickeys
|
||||
from onionrutils import bytesconverter
|
||||
from etc import onionrvalues
|
||||
from utils import reconstructhash
|
||||
from onionrcommands import restartonionr
|
||||
"""
|
||||
|
||||
pub_key = onionrcrypto.pub_key.replace('=', '')
|
||||
|
||||
SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + f'/../../../{onionrvalues.SCRIPT_NAME}'
|
||||
SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + \
|
||||
f'/../../../{onionrvalues.SCRIPT_NAME}'
|
||||
|
||||
|
||||
|
||||
class PrivateEndpoints:
|
||||
def __init__(self, client_api):
|
||||
|
@ -84,12 +96,30 @@ class PrivateEndpoints:
|
|||
|
||||
@private_endpoints_bp.route('/waitforshare/<name>', methods=['post'])
|
||||
def waitforshare(name):
|
||||
'''Used to prevent the **public** api from sharing blocks we just created'''
|
||||
if not name.isalnum(): raise ValueError('block hash needs to be alpha numeric')
|
||||
"""Prevent the **public** api from sharing blocks we just created"""
|
||||
def _delay_wait_for_share_block_removal(block: 'BlockHash'):
|
||||
min_w = onionrvalues.MIN_SHARE_WAIT_DELAY_SECS
|
||||
# Delay at least min but otherwise getBlocks timer + rand 10s
|
||||
delay_before_remove = max(
|
||||
min_w,
|
||||
randbelow
|
||||
(config.get
|
||||
(
|
||||
'timers.getBlocks',
|
||||
default=10) + randbelow(11)))
|
||||
sleep(delay_before_remove)
|
||||
try:
|
||||
client_api.publicAPI.hideBlocks.remove(name)
|
||||
except ValueError:
|
||||
logger.warn(
|
||||
f'Failed to remove {name} from waitforshare')
|
||||
|
||||
if not name.isalnum():
|
||||
raise ValueError('block hash needs to be alnum')
|
||||
name = reconstructhash.reconstruct_hash(name)
|
||||
if name in client_api.publicAPI.hideBlocks:
|
||||
client_api.publicAPI.hideBlocks.remove(name)
|
||||
return Response("removed")
|
||||
spawn(_delay_wait_for_share_block_removal)
|
||||
return Response("will be removed")
|
||||
else:
|
||||
client_api.publicAPI.hideBlocks.append(name)
|
||||
return Response("added")
|
||||
|
|
|
@ -1,3 +1,5 @@
|
|||
from . import insert
|
||||
from .insert import time_insert
|
||||
|
||||
insert = insert.insert_block
|
||||
insert = insert.insert_block
|
||||
time_insert = time_insert
|
|
@ -0,0 +1,4 @@
|
|||
from . import main, timeinsert
|
||||
|
||||
insert_block = main.insert_block
|
||||
time_insert = timeinsert.time_insert
|
|
@ -22,7 +22,7 @@ import json
|
|||
|
||||
from onionrutils import bytesconverter, epoch
|
||||
import filepaths, onionrstorage
|
||||
from . import storagecounter
|
||||
from .. import storagecounter
|
||||
from onionrplugins import onionrevents as events
|
||||
from etc import powchoice, onionrvalues
|
||||
import config, onionrcrypto as crypto, onionrexceptions
|
|
@ -0,0 +1,51 @@
|
|||
"""Onionr - Private P2P Communication.
|
||||
|
||||
Wrapper to insert blocks with variable delay
|
||||
"""
|
||||
from . import main
|
||||
"""
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
|
||||
|
||||
def time_insert(*args, **kwargs):
|
||||
"""Block insert wrapper to allow for insertions independent of mixmate.
|
||||
|
||||
Takes exact args as insert_block, with additional keyword:
|
||||
delay=n; where n=seconds to tell initial nodes to delay share for.
|
||||
|
||||
defaults to 0 or previously set value in current block meta
|
||||
"""
|
||||
try:
|
||||
kwargs['meta']
|
||||
except KeyError:
|
||||
kwargs['meta'] = {}
|
||||
|
||||
try:
|
||||
delay = int(kwargs['meta']['dly'])
|
||||
except KeyError:
|
||||
delay = 0
|
||||
try:
|
||||
delay = kwargs['delay']
|
||||
del kwargs['delay']
|
||||
except KeyError:
|
||||
delay = 0
|
||||
|
||||
# Ensure delay >=0
|
||||
if delay < 0:
|
||||
raise ValueError('delay cannot be less than 0')
|
||||
|
||||
kwargs['meta']['dly'] = delay
|
||||
|
||||
return main.insert_block(*args, **kwargs)
|
|
@ -0,0 +1,11 @@
|
|||
#!/usr/bin/env python3
|
||||
import unittest, sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("src/")
|
||||
|
||||
|
||||
class TestTemplate(unittest.TestCase):
|
||||
def test_my_test(self):
|
||||
self.assertTrue(True)
|
||||
|
||||
unittest.main()
|
|
@ -0,0 +1,28 @@
|
|||
#!/usr/bin/env python3
|
||||
import unittest, sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("src/")
|
||||
|
||||
from onionrblocks import time_insert
|
||||
from onionrblocks import onionrblockapi
|
||||
|
||||
class TestTimeInsert(unittest.TestCase):
|
||||
def test_time_insert_none(self):
|
||||
bl = time_insert('test')
|
||||
self.assertTrue(bl)
|
||||
bl = onionrblockapi.Block(bl)
|
||||
self.assertIs(bl.bmetadata['dly'], 0)
|
||||
|
||||
def test_time_insert_10(self):
|
||||
bl = time_insert('test', delay=10)
|
||||
self.assertTrue(bl)
|
||||
bl = onionrblockapi.Block(bl)
|
||||
self.assertIs(bl.bmetadata['dly'], 10)
|
||||
|
||||
def test_negative(self):
|
||||
self.assertRaises(ValueError, time_insert, 'test', delay=-1)
|
||||
self.assertRaises(ValueError, time_insert, 'test', delay=-10)
|
||||
|
||||
|
||||
|
||||
unittest.main()
|
Loading…
Reference in New Issue