work on block insertion mixing
parent
87ea8d137b
commit
01f9b9b470
|
@ -5,10 +5,12 @@ Handle daemon queue commands in the communicator
|
||||||
import logger
|
import logger
|
||||||
from onionrplugins import onionrevents as events
|
from onionrplugins import onionrevents as events
|
||||||
from onionrutils import localcommand
|
from onionrutils import localcommand
|
||||||
|
from communicatorutils.uploadblocks import mixmate
|
||||||
from coredb import daemonqueue
|
from coredb import daemonqueue
|
||||||
import filepaths
|
import filepaths
|
||||||
from . import restarttor
|
|
||||||
from communicatorutils.uploadblocks import mixmate
|
from .. import restarttor
|
||||||
|
|
||||||
"""
|
"""
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
|
@ -33,7 +35,8 @@ def handle_daemon_commands(comm_inst):
|
||||||
if cmd[0] == 'shutdown':
|
if cmd[0] == 'shutdown':
|
||||||
comm_inst.shutdown = True
|
comm_inst.shutdown = True
|
||||||
elif cmd[0] == 'runtimeTest':
|
elif cmd[0] == 'runtimeTest':
|
||||||
comm_inst.shared_state.get_by_string("OnionrRunTestManager").run_tests()
|
comm_inst.shared_state.get_by_string(
|
||||||
|
"OnionrRunTestManager").run_tests()
|
||||||
elif cmd[0] == 'remove_from_insert_list':
|
elif cmd[0] == 'remove_from_insert_list':
|
||||||
try:
|
try:
|
||||||
comm_inst.generating_blocks.remove(cmd[1])
|
comm_inst.generating_blocks.remove(cmd[1])
|
||||||
|
@ -65,18 +68,21 @@ def handle_daemon_commands(comm_inst):
|
||||||
elif cmd[0] == 'uploadBlock':
|
elif cmd[0] == 'uploadBlock':
|
||||||
comm_inst.blocksToUpload.append(cmd[1])
|
comm_inst.blocksToUpload.append(cmd[1])
|
||||||
elif cmd[0] == 'uploadEvent':
|
elif cmd[0] == 'uploadEvent':
|
||||||
|
localcommand.local_command('/waitforshare/' + cmd[1], post=True,
|
||||||
|
maxWait=5)
|
||||||
try:
|
try:
|
||||||
mixmate.block_mixer(comm_inst.blocksToUpload, cmd[1])
|
mixmate.block_mixer(comm_inst.blocksToUpload, cmd[1])
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
comm_inst.blocksToUpload.append(cmd[1])
|
||||||
else:
|
else:
|
||||||
localcommand.local_command('/waitforshare/' + cmd[1], post=True, maxWait=5)
|
logger.debug(
|
||||||
else:
|
'Received daemon queue cmd with no handler: %s' % (cmd[0],))
|
||||||
logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],))
|
|
||||||
|
|
||||||
if cmd[0] not in ('', None):
|
if cmd[0] not in ('', None):
|
||||||
if response != '':
|
if response != '':
|
||||||
localcommand.local_command('queueResponseAdd/' + cmd[4], post=True, postData={'data': response})
|
localcommand.local_command('queueResponseAdd/' + cmd[4],
|
||||||
|
post=True,
|
||||||
|
postData={'data': response})
|
||||||
response = ''
|
response = ''
|
||||||
|
|
||||||
comm_inst.decrementThreadCount('handle_daemon_commands')
|
comm_inst.decrementThreadCount('handle_daemon_commands')
|
|
@ -15,7 +15,6 @@ import onionrcrypto
|
||||||
from communicator import onlinepeers
|
from communicator import onlinepeers
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from communicator import OnionrCommunicatorDaemon
|
from communicator import OnionrCommunicatorDaemon
|
||||||
|
|
||||||
"""
|
"""
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
|
|
|
@ -38,6 +38,14 @@ def block_mixer(upload_list: List[onionrtypes.BlockHash],
|
||||||
to the said block list
|
to the said block list
|
||||||
"""
|
"""
|
||||||
bl = onionrblockapi.Block(block_to_mix)
|
bl = onionrblockapi.Block(block_to_mix)
|
||||||
|
|
||||||
|
try:
|
||||||
|
bl.bmetadata['dly']
|
||||||
|
except (KeyError, TypeError):
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
raise ValueError
|
||||||
|
|
||||||
if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE:
|
if time.time() - bl.claimedTime > onionrvalues.BLOCK_POOL_MAX_AGE:
|
||||||
raise ValueError
|
raise ValueError
|
||||||
|
|
||||||
|
|
|
@ -106,7 +106,10 @@ class BlockUploadSessionManager:
|
||||||
if (sess.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
|
if (sess.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
|
||||||
sessions_to_delete.append(sess)
|
sessions_to_delete.append(sess)
|
||||||
for sess in sessions_to_delete:
|
for sess in sessions_to_delete:
|
||||||
|
try:
|
||||||
self.sessions.remove(session)
|
self.sessions.remove(session)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
# TODO cleanup to one round of search
|
# TODO cleanup to one round of search
|
||||||
# Remove the blocks from the sessions, upload list,
|
# Remove the blocks from the sessions, upload list,
|
||||||
# and waitforshare list
|
# and waitforshare list
|
||||||
|
|
|
@ -43,6 +43,7 @@ DATABASE_LOCK_TIMEOUT = 60
|
||||||
|
|
||||||
# Block creation anonymization requirements
|
# Block creation anonymization requirements
|
||||||
MIN_BLOCK_UPLOAD_PEER_PERCENT = 0.1
|
MIN_BLOCK_UPLOAD_PEER_PERCENT = 0.1
|
||||||
|
MIN_SHARE_WAIT_DELAY_SECS = 5
|
||||||
|
|
||||||
# Begin OnionrValues migrated values
|
# Begin OnionrValues migrated values
|
||||||
"""Make announce take a few seconds (on average) to compute to discourage excessive node announcements"""
|
"""Make announce take a few seconds (on average) to compute to discourage excessive node announcements"""
|
||||||
|
|
|
@ -1,9 +1,33 @@
|
||||||
'''
|
"""Onionr - Private P2P Communication.
|
||||||
Onionr - Private P2P Communication
|
|
||||||
|
|
||||||
Misc client API endpoints too small to need their own file and that need access to the client api inst
|
Misc client API endpoints too small to need their own file
|
||||||
'''
|
and that need access to the client api inst
|
||||||
'''
|
"""
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
from secrets import randbelow
|
||||||
|
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
from flask import Response, Blueprint, request, send_from_directory, abort
|
||||||
|
from gevent import spawn
|
||||||
|
from gevent import sleep
|
||||||
|
import unpaddedbase32
|
||||||
|
|
||||||
|
from httpapi import apiutils
|
||||||
|
import logger
|
||||||
|
import onionrcrypto
|
||||||
|
import config
|
||||||
|
from netcontroller import NetController
|
||||||
|
from serializeddata import SerializedData
|
||||||
|
from onionrutils import mnemonickeys
|
||||||
|
from onionrutils import bytesconverter
|
||||||
|
from etc import onionrvalues
|
||||||
|
from utils import reconstructhash
|
||||||
|
from onionrcommands import restartonionr
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from onionrtypes import BlockHash
|
||||||
|
"""
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
@ -16,26 +40,14 @@
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
You should have received a copy of the GNU General Public License
|
||||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
'''
|
"""
|
||||||
import os
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
from flask import Response, Blueprint, request, send_from_directory, abort
|
|
||||||
import unpaddedbase32
|
|
||||||
|
|
||||||
from httpapi import apiutils
|
|
||||||
import onionrcrypto, config
|
|
||||||
from netcontroller import NetController
|
|
||||||
from serializeddata import SerializedData
|
|
||||||
from onionrutils import mnemonickeys
|
|
||||||
from onionrutils import bytesconverter
|
|
||||||
from etc import onionrvalues
|
|
||||||
from utils import reconstructhash
|
|
||||||
from onionrcommands import restartonionr
|
|
||||||
|
|
||||||
pub_key = onionrcrypto.pub_key.replace('=', '')
|
pub_key = onionrcrypto.pub_key.replace('=', '')
|
||||||
|
|
||||||
SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + f'/../../../{onionrvalues.SCRIPT_NAME}'
|
SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + \
|
||||||
|
f'/../../../{onionrvalues.SCRIPT_NAME}'
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class PrivateEndpoints:
|
class PrivateEndpoints:
|
||||||
def __init__(self, client_api):
|
def __init__(self, client_api):
|
||||||
|
@ -84,12 +96,30 @@ class PrivateEndpoints:
|
||||||
|
|
||||||
@private_endpoints_bp.route('/waitforshare/<name>', methods=['post'])
|
@private_endpoints_bp.route('/waitforshare/<name>', methods=['post'])
|
||||||
def waitforshare(name):
|
def waitforshare(name):
|
||||||
'''Used to prevent the **public** api from sharing blocks we just created'''
|
"""Prevent the **public** api from sharing blocks we just created"""
|
||||||
if not name.isalnum(): raise ValueError('block hash needs to be alpha numeric')
|
def _delay_wait_for_share_block_removal(block: 'BlockHash'):
|
||||||
|
min_w = onionrvalues.MIN_SHARE_WAIT_DELAY_SECS
|
||||||
|
# Delay at least min but otherwise getBlocks timer + rand 10s
|
||||||
|
delay_before_remove = max(
|
||||||
|
min_w,
|
||||||
|
randbelow
|
||||||
|
(config.get
|
||||||
|
(
|
||||||
|
'timers.getBlocks',
|
||||||
|
default=10) + randbelow(11)))
|
||||||
|
sleep(delay_before_remove)
|
||||||
|
try:
|
||||||
|
client_api.publicAPI.hideBlocks.remove(name)
|
||||||
|
except ValueError:
|
||||||
|
logger.warn(
|
||||||
|
f'Failed to remove {name} from waitforshare')
|
||||||
|
|
||||||
|
if not name.isalnum():
|
||||||
|
raise ValueError('block hash needs to be alnum')
|
||||||
name = reconstructhash.reconstruct_hash(name)
|
name = reconstructhash.reconstruct_hash(name)
|
||||||
if name in client_api.publicAPI.hideBlocks:
|
if name in client_api.publicAPI.hideBlocks:
|
||||||
client_api.publicAPI.hideBlocks.remove(name)
|
spawn(_delay_wait_for_share_block_removal)
|
||||||
return Response("removed")
|
return Response("will be removed")
|
||||||
else:
|
else:
|
||||||
client_api.publicAPI.hideBlocks.append(name)
|
client_api.publicAPI.hideBlocks.append(name)
|
||||||
return Response("added")
|
return Response("added")
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
from . import insert
|
from . import insert
|
||||||
|
from .insert import time_insert
|
||||||
|
|
||||||
insert = insert.insert_block
|
insert = insert.insert_block
|
||||||
|
time_insert = time_insert
|
|
@ -0,0 +1,4 @@
|
||||||
|
from . import main, timeinsert
|
||||||
|
|
||||||
|
insert_block = main.insert_block
|
||||||
|
time_insert = timeinsert.time_insert
|
|
@ -22,7 +22,7 @@ import json
|
||||||
|
|
||||||
from onionrutils import bytesconverter, epoch
|
from onionrutils import bytesconverter, epoch
|
||||||
import filepaths, onionrstorage
|
import filepaths, onionrstorage
|
||||||
from . import storagecounter
|
from .. import storagecounter
|
||||||
from onionrplugins import onionrevents as events
|
from onionrplugins import onionrevents as events
|
||||||
from etc import powchoice, onionrvalues
|
from etc import powchoice, onionrvalues
|
||||||
import config, onionrcrypto as crypto, onionrexceptions
|
import config, onionrcrypto as crypto, onionrexceptions
|
|
@ -0,0 +1,51 @@
|
||||||
|
"""Onionr - Private P2P Communication.
|
||||||
|
|
||||||
|
Wrapper to insert blocks with variable delay
|
||||||
|
"""
|
||||||
|
from . import main
|
||||||
|
"""
|
||||||
|
This program is free software: you can redistribute it and/or modify
|
||||||
|
it under the terms of the GNU General Public License as published by
|
||||||
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
(at your option) any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License
|
||||||
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def time_insert(*args, **kwargs):
|
||||||
|
"""Block insert wrapper to allow for insertions independent of mixmate.
|
||||||
|
|
||||||
|
Takes exact args as insert_block, with additional keyword:
|
||||||
|
delay=n; where n=seconds to tell initial nodes to delay share for.
|
||||||
|
|
||||||
|
defaults to 0 or previously set value in current block meta
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
kwargs['meta']
|
||||||
|
except KeyError:
|
||||||
|
kwargs['meta'] = {}
|
||||||
|
|
||||||
|
try:
|
||||||
|
delay = int(kwargs['meta']['dly'])
|
||||||
|
except KeyError:
|
||||||
|
delay = 0
|
||||||
|
try:
|
||||||
|
delay = kwargs['delay']
|
||||||
|
del kwargs['delay']
|
||||||
|
except KeyError:
|
||||||
|
delay = 0
|
||||||
|
|
||||||
|
# Ensure delay >=0
|
||||||
|
if delay < 0:
|
||||||
|
raise ValueError('delay cannot be less than 0')
|
||||||
|
|
||||||
|
kwargs['meta']['dly'] = delay
|
||||||
|
|
||||||
|
return main.insert_block(*args, **kwargs)
|
|
@ -0,0 +1,11 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import unittest, sys
|
||||||
|
sys.path.append(".")
|
||||||
|
sys.path.append("src/")
|
||||||
|
|
||||||
|
|
||||||
|
class TestTemplate(unittest.TestCase):
|
||||||
|
def test_my_test(self):
|
||||||
|
self.assertTrue(True)
|
||||||
|
|
||||||
|
unittest.main()
|
|
@ -0,0 +1,28 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import unittest, sys
|
||||||
|
sys.path.append(".")
|
||||||
|
sys.path.append("src/")
|
||||||
|
|
||||||
|
from onionrblocks import time_insert
|
||||||
|
from onionrblocks import onionrblockapi
|
||||||
|
|
||||||
|
class TestTimeInsert(unittest.TestCase):
|
||||||
|
def test_time_insert_none(self):
|
||||||
|
bl = time_insert('test')
|
||||||
|
self.assertTrue(bl)
|
||||||
|
bl = onionrblockapi.Block(bl)
|
||||||
|
self.assertIs(bl.bmetadata['dly'], 0)
|
||||||
|
|
||||||
|
def test_time_insert_10(self):
|
||||||
|
bl = time_insert('test', delay=10)
|
||||||
|
self.assertTrue(bl)
|
||||||
|
bl = onionrblockapi.Block(bl)
|
||||||
|
self.assertIs(bl.bmetadata['dly'], 10)
|
||||||
|
|
||||||
|
def test_negative(self):
|
||||||
|
self.assertRaises(ValueError, time_insert, 'test', delay=-1)
|
||||||
|
self.assertRaises(ValueError, time_insert, 'test', delay=-10)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
unittest.main()
|
Loading…
Reference in New Issue