minor bug fixes and linting improvements
parent
9329b07e3b
commit
87ea8d137b
|
@ -1,9 +1,10 @@
|
||||||
'''
|
"""Onionr - Private P2P Communication.
|
||||||
Onionr - Private P2P Communication
|
|
||||||
|
|
||||||
add bootstrap peers to the communicator peer list
|
add bootstrap peers to the communicator peer list
|
||||||
'''
|
"""
|
||||||
'''
|
from utils import readstatic, gettransports
|
||||||
|
from coredb import keydb
|
||||||
|
"""
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
@ -16,16 +17,16 @@
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
You should have received a copy of the GNU General Public License
|
||||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
'''
|
"""
|
||||||
from utils import readstatic, gettransports
|
|
||||||
from coredb import keydb
|
|
||||||
bootstrap_peers = readstatic.read_static('bootstrap-nodes.txt').split(',')
|
bootstrap_peers = readstatic.read_static('bootstrap-nodes.txt').split(',')
|
||||||
|
|
||||||
|
|
||||||
def add_bootstrap_list_to_peer_list(comm_inst, peerList, db_only=False):
|
def add_bootstrap_list_to_peer_list(comm_inst, peerList, db_only=False):
|
||||||
'''
|
"""Add the bootstrap list to the peer list (no duplicates)."""
|
||||||
Add the bootstrap list to the peer list (no duplicates)
|
|
||||||
'''
|
|
||||||
for i in bootstrap_peers:
|
for i in bootstrap_peers:
|
||||||
if i not in peerList and i not in comm_inst.offlinePeers and not i in gettransports.get() and len(str(i).strip()) > 0:
|
if i not in peerList and i not in comm_inst.offlinePeers \
|
||||||
if not db_only: peerList.append(i)
|
and i not in gettransports.get() and len(str(i).strip()) > 0:
|
||||||
|
if not db_only:
|
||||||
|
peerList.append(i)
|
||||||
keydb.addkeys.add_address(i)
|
keydb.addkeys.add_address(i)
|
||||||
|
|
|
@ -1,9 +1,13 @@
|
||||||
'''
|
"""Onionr - Private P2P Communication.
|
||||||
Onionr - Private P2P Communication
|
|
||||||
|
|
||||||
This file implements logic for performing requests to Onionr peers
|
This file implements logic for performing requests to Onionr peers
|
||||||
'''
|
"""
|
||||||
'''
|
import streamedrequests
|
||||||
|
import logger
|
||||||
|
from onionrutils import epoch, basicrequests
|
||||||
|
from coredb import keydb
|
||||||
|
from . import onlinepeers
|
||||||
|
"""
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
@ -16,15 +20,12 @@
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
You should have received a copy of the GNU General Public License
|
||||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
'''
|
"""
|
||||||
import streamedrequests
|
|
||||||
import logger
|
|
||||||
from onionrutils import epoch, basicrequests
|
|
||||||
from coredb import keydb
|
|
||||||
from . import onlinepeers
|
|
||||||
|
|
||||||
def peer_action(comm_inst, peer, action, returnHeaders=False, max_resp_size=5242880):
|
|
||||||
'''Perform a get request to a peer'''
|
def peer_action(comm_inst, peer, action,
|
||||||
|
returnHeaders=False, max_resp_size=5242880):
|
||||||
|
"""Perform a get request to a peer."""
|
||||||
penalty_score = -10
|
penalty_score = -10
|
||||||
if len(peer) == 0:
|
if len(peer) == 0:
|
||||||
return False
|
return False
|
||||||
|
@ -34,22 +35,28 @@ def peer_action(comm_inst, peer, action, returnHeaders=False, max_resp_size=5242
|
||||||
ret_data = basicrequests.do_get_request(url, port=comm_inst.proxyPort,
|
ret_data = basicrequests.do_get_request(url, port=comm_inst.proxyPort,
|
||||||
max_size=max_resp_size)
|
max_size=max_resp_size)
|
||||||
except streamedrequests.exceptions.ResponseLimitReached:
|
except streamedrequests.exceptions.ResponseLimitReached:
|
||||||
logger.warn('Request failed due to max response size being overflowed', terminal=True)
|
logger.warn(
|
||||||
|
'Request failed due to max response size being overflowed',
|
||||||
|
terminal=True)
|
||||||
ret_data = False
|
ret_data = False
|
||||||
penalty_score = -100
|
penalty_score = -100
|
||||||
# if request failed, (error), mark peer offline
|
# if request failed, (error), mark peer offline
|
||||||
if ret_data == False: # For some reason "if not" breaks this. Prob has to do with empty string.
|
if ret_data is False:
|
||||||
try:
|
try:
|
||||||
comm_inst.getPeerProfileInstance(peer).addScore(penalty_score)
|
comm_inst.getPeerProfileInstance(peer).addScore(penalty_score)
|
||||||
onlinepeers.remove_online_peer(comm_inst, peer)
|
onlinepeers.remove_online_peer(comm_inst, peer)
|
||||||
keydb.transportinfo.set_address_info(peer, 'lastConnectAttempt', epoch.get_epoch())
|
keydb.transportinfo.set_address_info(
|
||||||
|
peer, 'lastConnectAttempt', epoch.get_epoch())
|
||||||
if action != 'ping' and not comm_inst.shutdown:
|
if action != 'ping' and not comm_inst.shutdown:
|
||||||
logger.warn(f'Lost connection to {peer}', terminal=True)
|
logger.warn(f'Lost connection to {peer}', terminal=True)
|
||||||
onlinepeers.get_online_peers(comm_inst) # Will only add a new peer to pool if needed
|
# Will only add a new peer to pool if needed
|
||||||
|
onlinepeers.get_online_peers(comm_inst)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
else:
|
else:
|
||||||
peer_profile = comm_inst.getPeerProfileInstance(peer)
|
peer_profile = comm_inst.getPeerProfileInstance(peer)
|
||||||
peer_profile.update_connect_time()
|
peer_profile.update_connect_time()
|
||||||
peer_profile.addScore(1)
|
peer_profile.addScore(1)
|
||||||
return ret_data # If returnHeaders, returns tuple of data, headers. if not, just data string
|
# If returnHeaders, returns tuple of data, headers.
|
||||||
|
# If not, just data string
|
||||||
|
return ret_data
|
||||||
|
|
|
@ -1,9 +1,20 @@
|
||||||
'''
|
"""
|
||||||
Onionr - Private P2P Communication
|
Onionr - Private P2P Communication.
|
||||||
|
|
||||||
Use a communicator instance to announce our transport address to connected nodes
|
Use a communicator instance to announce
|
||||||
'''
|
our transport address to connected nodes
|
||||||
'''
|
"""
|
||||||
|
import base64
|
||||||
|
import onionrproofs
|
||||||
|
import logger
|
||||||
|
from etc import onionrvalues
|
||||||
|
from onionrutils import basicrequests, bytesconverter
|
||||||
|
from utils import gettransports
|
||||||
|
from netcontroller import NetController
|
||||||
|
from communicator import onlinepeers
|
||||||
|
from coredb import keydb
|
||||||
|
import onionrexceptions
|
||||||
|
"""
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
@ -16,22 +27,14 @@
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
You should have received a copy of the GNU General Public License
|
||||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
'''
|
"""
|
||||||
import base64
|
|
||||||
import onionrproofs, logger
|
|
||||||
from etc import onionrvalues
|
|
||||||
from onionrutils import basicrequests, bytesconverter
|
|
||||||
from utils import gettransports
|
|
||||||
from netcontroller import NetController
|
|
||||||
from communicator import onlinepeers
|
|
||||||
from coredb import keydb
|
|
||||||
import onionrexceptions
|
|
||||||
|
|
||||||
def announce_node(daemon):
|
def announce_node(daemon):
|
||||||
'''Announce our node to our peers'''
|
"""Announce our node to our peers."""
|
||||||
ret_data = False
|
ret_data = False
|
||||||
announce_fail = False
|
announce_fail = False
|
||||||
|
|
||||||
# Do not let announceCache get too large
|
# Do not let announceCache get too large
|
||||||
if len(daemon.announceCache) >= 10000:
|
if len(daemon.announceCache) >= 10000:
|
||||||
daemon.announceCache.popitem()
|
daemon.announceCache.popitem()
|
||||||
|
@ -39,7 +42,8 @@ def announce_node(daemon):
|
||||||
if daemon.config.get('general.security_level', 0) == 0:
|
if daemon.config.get('general.security_level', 0) == 0:
|
||||||
# Announce to random online peers
|
# Announce to random online peers
|
||||||
for i in daemon.onlinePeers:
|
for i in daemon.onlinePeers:
|
||||||
if not i in daemon.announceCache and not i in daemon.announceProgress:
|
if i not in daemon.announceCache and\
|
||||||
|
i not in daemon.announceProgress:
|
||||||
peer = i
|
peer = i
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
|
@ -61,9 +65,12 @@ def announce_node(daemon):
|
||||||
|
|
||||||
combinedNodes = ourID + peer
|
combinedNodes = ourID + peer
|
||||||
if ourID != 1:
|
if ourID != 1:
|
||||||
existingRand = bytesconverter.bytes_to_str(keydb.transportinfo.get_address_info(peer, 'powValue'))
|
existingRand = bytesconverter.bytes_to_str(
|
||||||
|
keydb.transportinfo.get_address_info(peer, 'powValue'))
|
||||||
# Reset existingRand if it no longer meets the minimum POW
|
# Reset existingRand if it no longer meets the minimum POW
|
||||||
if type(existingRand) is type(None) or not existingRand.endswith('0' * onionrvalues.ANNOUNCE_POW):
|
if isinstance(existingRand, type(None)) or \
|
||||||
|
not existingRand.endswith(
|
||||||
|
'0' * onionrvalues.ANNOUNCE_POW):
|
||||||
existingRand = ''
|
existingRand = ''
|
||||||
|
|
||||||
if peer in daemon.announceCache:
|
if peer in daemon.announceCache:
|
||||||
|
@ -72,22 +79,29 @@ def announce_node(daemon):
|
||||||
data['random'] = existingRand
|
data['random'] = existingRand
|
||||||
else:
|
else:
|
||||||
daemon.announceProgress[peer] = True
|
daemon.announceProgress[peer] = True
|
||||||
proof = onionrproofs.DataPOW(combinedNodes, minDifficulty=onionrvalues.ANNOUNCE_POW)
|
proof = onionrproofs.DataPOW(
|
||||||
|
combinedNodes, minDifficulty=onionrvalues.ANNOUNCE_POW)
|
||||||
del daemon.announceProgress[peer]
|
del daemon.announceProgress[peer]
|
||||||
try:
|
try:
|
||||||
data['random'] = base64.b64encode(proof.waitForResult()[1])
|
data['random'] = base64.b64encode(proof.waitForResult()[1])
|
||||||
except TypeError:
|
except TypeError:
|
||||||
# Happens when we failed to produce a proof
|
# Happens when we failed to produce a proof
|
||||||
logger.error("Failed to produce a pow for announcing to " + peer)
|
logger.error(f"Failed to produce a pow for {peer} annce")
|
||||||
announce_fail = True
|
announce_fail = True
|
||||||
else:
|
else:
|
||||||
daemon.announceCache[peer] = data['random']
|
daemon.announceCache[peer] = data['random']
|
||||||
if not announce_fail:
|
if not announce_fail:
|
||||||
logger.info('Announcing node to ' + url)
|
logger.info('Announcing node to ' + url)
|
||||||
if basicrequests.do_post_request(url, data, port=daemon.shared_state.get(NetController).socksPort) == 'Success':
|
if basicrequests.do_post_request(
|
||||||
logger.info('Successfully introduced node to ' + peer, terminal=True)
|
url,
|
||||||
|
data,
|
||||||
|
port=daemon.shared_state.get(NetController).socksPort)\
|
||||||
|
== 'Success':
|
||||||
|
logger.info('Successfully introduced node to ' + peer,
|
||||||
|
terminal=True)
|
||||||
ret_data = True
|
ret_data = True
|
||||||
keydb.transportinfo.set_address_info(peer, 'introduced', 1)
|
keydb.transportinfo.set_address_info(peer, 'introduced', 1)
|
||||||
keydb.transportinfo.set_address_info(peer, 'powValue', data['random'])
|
keydb.transportinfo.set_address_info(peer, 'powValue',
|
||||||
|
data['random'])
|
||||||
daemon.decrementThreadCount('announce_node')
|
daemon.decrementThreadCount('announce_node')
|
||||||
return ret_data
|
return ret_data
|
||||||
|
|
|
@ -1,9 +1,11 @@
|
||||||
'''
|
"""Onionr - Private P2P Communication.
|
||||||
Onionr - Private P2P Communication
|
|
||||||
|
|
||||||
Check if a block should be downloaded (if we already have it or its blacklisted or not)
|
Check if a block should be downloaded
|
||||||
'''
|
(if we already have it or its blacklisted or not)
|
||||||
'''
|
"""
|
||||||
|
from coredb import blockmetadb
|
||||||
|
from onionrblocks import onionrblacklist
|
||||||
|
"""
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
@ -16,22 +18,24 @@
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
You should have received a copy of the GNU General Public License
|
||||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
'''
|
"""
|
||||||
from coredb import blockmetadb
|
|
||||||
from onionrblocks import onionrblacklist
|
|
||||||
|
|
||||||
def should_download(comm_inst, block_hash):
|
|
||||||
|
def should_download(comm_inst, block_hash) -> bool:
|
||||||
|
"""Return bool for if a (assumed to exist) block should be downloaded."""
|
||||||
blacklist = onionrblacklist.OnionrBlackList()
|
blacklist = onionrblacklist.OnionrBlackList()
|
||||||
ret_data = True
|
should = True
|
||||||
if block_hash in blockmetadb.get_block_list(): # Dont download block we have
|
if block_hash in blockmetadb.get_block_list():
|
||||||
ret_data = False
|
# Don't download block we have
|
||||||
|
should = False
|
||||||
else:
|
else:
|
||||||
if blacklist.inBlacklist(block_hash): # Dont download blacklisted block
|
if blacklist.inBlacklist(block_hash):
|
||||||
ret_data = False
|
# Don't download blacklisted block
|
||||||
if ret_data is False:
|
should = False
|
||||||
# Remove block from communicator queue if it shouldnt be downloaded
|
if should is False:
|
||||||
|
# Remove block from communicator queue if it shouldn't be downloaded
|
||||||
try:
|
try:
|
||||||
del comm_inst.blockQueue[block_hash]
|
del comm_inst.blockQueue[block_hash]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
pass
|
pass
|
||||||
return ret_data
|
return should
|
||||||
|
|
|
@ -1,10 +1,22 @@
|
||||||
'''
|
"""Onionr - Private P2P Communication.
|
||||||
Onionr - Private P2P Communication
|
|
||||||
|
|
||||||
Upload blocks in the upload queue to peers from the communicator
|
Upload blocks in the upload queue to peers from the communicator
|
||||||
'''
|
"""
|
||||||
from __future__ import annotations
|
from typing import TYPE_CHECKING
|
||||||
'''
|
from . import sessionmanager
|
||||||
|
|
||||||
|
from onionrtypes import UserID
|
||||||
|
import logger
|
||||||
|
from communicatorutils import proxypicker
|
||||||
|
import onionrexceptions
|
||||||
|
from onionrblocks import onionrblockapi as block
|
||||||
|
from onionrutils import stringvalidators, basicrequests
|
||||||
|
import onionrcrypto
|
||||||
|
from communicator import onlinepeers
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from communicator import OnionrCommunicatorDaemon
|
||||||
|
|
||||||
|
"""
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
@ -17,28 +29,23 @@ from __future__ import annotations
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
You should have received a copy of the GNU General Public License
|
||||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
'''
|
"""
|
||||||
from typing import Union, TYPE_CHECKING
|
|
||||||
import logger
|
|
||||||
from communicatorutils import proxypicker
|
|
||||||
import onionrexceptions
|
|
||||||
from onionrblocks import onionrblockapi as block
|
|
||||||
from onionrutils import localcommand, stringvalidators, basicrequests
|
|
||||||
from communicator import onlinepeers
|
|
||||||
import onionrcrypto
|
|
||||||
from . import sessionmanager
|
|
||||||
from . import mixmate
|
|
||||||
|
|
||||||
def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
|
|
||||||
"""Accepts a communicator instance and uploads blocks from its upload queue"""
|
def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
|
||||||
|
"""Accept a communicator instance + upload blocks from its upload queue."""
|
||||||
"""when inserting a block, we try to upload
|
"""when inserting a block, we try to upload
|
||||||
it to a few peers to add some deniability & increase functionality"""
|
it to a few peers to add some deniability & increase functionality"""
|
||||||
TIMER_NAME = "upload_blocks_from_communicator"
|
TIMER_NAME = "upload_blocks_from_communicator"
|
||||||
|
|
||||||
session_manager: sessionmanager.BlockUploadSessionManager = comm_inst.shared_state.get(sessionmanager.BlockUploadSessionManager)
|
session_manager: sessionmanager.BlockUploadSessionManager
|
||||||
triedPeers = []
|
session_manager = comm_inst.shared_state.get(
|
||||||
|
sessionmanager.BlockUploadSessionManager)
|
||||||
|
tried_peers: UserID = []
|
||||||
finishedUploads = []
|
finishedUploads = []
|
||||||
comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(comm_inst.blocksToUpload)
|
comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(
|
||||||
|
comm_inst.blocksToUpload)
|
||||||
|
|
||||||
if len(comm_inst.blocksToUpload) != 0:
|
if len(comm_inst.blocksToUpload) != 0:
|
||||||
for bl in comm_inst.blocksToUpload:
|
for bl in comm_inst.blocksToUpload:
|
||||||
if not stringvalidators.validate_hash(bl):
|
if not stringvalidators.validate_hash(bl):
|
||||||
|
@ -57,21 +64,26 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
|
||||||
except KeyError:
|
except KeyError:
|
||||||
pass
|
pass
|
||||||
try:
|
try:
|
||||||
if session.peer_fails[peer] > 3: continue
|
if session.peer_fails[peer] > 3:
|
||||||
|
continue
|
||||||
except KeyError:
|
except KeyError:
|
||||||
pass
|
pass
|
||||||
if peer in triedPeers: continue
|
if peer in tried_peers:
|
||||||
triedPeers.append(peer)
|
continue
|
||||||
|
tried_peers.append(peer)
|
||||||
url = f'http://{peer}/upload'
|
url = f'http://{peer}/upload'
|
||||||
try:
|
try:
|
||||||
data = block.Block(bl).getRaw()
|
data = block.Block(bl).getRaw()
|
||||||
except onionrexceptions.NoDataAvailable:
|
except onionrexceptions.NoDataAvailable:
|
||||||
finishedUploads.append(bl)
|
finishedUploads.append(bl)
|
||||||
break
|
break
|
||||||
proxyType = proxypicker.pick_proxy(peer)
|
proxy_type = proxypicker.pick_proxy(peer)
|
||||||
logger.info(f"Uploading block {bl[:8]} to {peer}", terminal=True)
|
logger.info(
|
||||||
resp = basicrequests.do_post_request(url, data=data, proxyType=proxyType, content_type='application/octet-stream')
|
f"Uploading block {bl[:8]} to {peer}", terminal=True)
|
||||||
if not resp == False:
|
resp = basicrequests.do_post_request(
|
||||||
|
url, data=data, proxyType=proxy_type,
|
||||||
|
content_type='application/octet-stream')
|
||||||
|
if resp is not False:
|
||||||
if resp == 'success':
|
if resp == 'success':
|
||||||
session.success()
|
session.success()
|
||||||
session.peer_exists[peer] = True
|
session.peer_exists[peer] = True
|
||||||
|
@ -82,7 +94,9 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
|
||||||
session.fail()
|
session.fail()
|
||||||
session.fail_peer(peer)
|
session.fail_peer(peer)
|
||||||
comm_inst.getPeerProfileInstance(peer).addScore(-5)
|
comm_inst.getPeerProfileInstance(peer).addScore(-5)
|
||||||
logger.warn(f'Failed to upload {bl[:8]}, reason: {resp[:15]}', terminal=True)
|
logger.warn(
|
||||||
|
f'Failed to upload {bl[:8]}, reason: {resp[:15]}',
|
||||||
|
terminal=True)
|
||||||
else:
|
else:
|
||||||
session.fail()
|
session.fail()
|
||||||
session_manager.clean_session()
|
session_manager.clean_session()
|
||||||
|
|
|
@ -62,7 +62,8 @@ class UploadPool:
|
||||||
"""Get the hash pool in secure random order."""
|
"""Get the hash pool in secure random order."""
|
||||||
if len(self._pool) != self._pool_size:
|
if len(self._pool) != self._pool_size:
|
||||||
raise PoolNotReady
|
raise PoolNotReady
|
||||||
final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(list(self._pool))
|
final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(
|
||||||
|
list(self._pool))
|
||||||
|
|
||||||
self._pool.clear()
|
self._pool.clear()
|
||||||
self.birthday = onionrutils.epoch.get_epoch()
|
self.birthday = onionrutils.epoch.get_epoch()
|
||||||
|
|
|
@ -2,8 +2,9 @@
|
||||||
|
|
||||||
Virtual upload "sessions" for blocks
|
Virtual upload "sessions" for blocks
|
||||||
"""
|
"""
|
||||||
from typing import Union
|
from typing import Union, Dict
|
||||||
|
|
||||||
|
from onionrtypes import UserID
|
||||||
from onionrutils import stringvalidators
|
from onionrutils import stringvalidators
|
||||||
from onionrutils import bytesconverter
|
from onionrutils import bytesconverter
|
||||||
from onionrutils import epoch
|
from onionrutils import epoch
|
||||||
|
@ -40,8 +41,8 @@ class UploadSession:
|
||||||
self.block_hash = reconstructhash.deconstruct_hash(block_hash)
|
self.block_hash = reconstructhash.deconstruct_hash(block_hash)
|
||||||
self.total_fail_count: int = 0
|
self.total_fail_count: int = 0
|
||||||
self.total_success_count: int = 0
|
self.total_success_count: int = 0
|
||||||
self.peer_fails = {}
|
self.peer_fails: Dict[UserID, int] = {}
|
||||||
self.peer_exists = {}
|
self.peer_exists: Dict[UserID, bool] = {}
|
||||||
|
|
||||||
def fail_peer(self, peer):
|
def fail_peer(self, peer):
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -1,9 +1,17 @@
|
||||||
"""
|
"""Onionr - Private P2P Communication.
|
||||||
Onionr - Private P2P Communication
|
|
||||||
|
|
||||||
Manager for upload 'sessions'
|
Manager for upload 'sessions'
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from typing import List, Union, TYPE_CHECKING
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from session import UploadSession
|
||||||
|
|
||||||
|
from onionrutils import bytesconverter
|
||||||
|
from onionrutils import localcommand
|
||||||
|
from etc import onionrvalues
|
||||||
|
from utils import reconstructhash
|
||||||
|
|
||||||
|
from . import session
|
||||||
"""
|
"""
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
|
@ -18,31 +26,33 @@ from __future__ import annotations
|
||||||
You should have received a copy of the GNU General Public License
|
You should have received a copy of the GNU General Public License
|
||||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
"""
|
"""
|
||||||
from typing import Iterable, Union
|
|
||||||
|
|
||||||
from onionrutils import bytesconverter
|
|
||||||
from onionrutils import localcommand
|
|
||||||
from etc import onionrvalues
|
|
||||||
from etc import waitforsetvar
|
|
||||||
from utils import reconstructhash
|
|
||||||
|
|
||||||
from . import session
|
|
||||||
|
|
||||||
class BlockUploadSessionManager:
|
class BlockUploadSessionManager:
|
||||||
"""Holds block UploadSession instances. Optionally accepts iterable of sessions to added on init
|
"""Holds block UploadSession instances.
|
||||||
|
|
||||||
Arguments: old_session: iterable of old UploadSession objects"""
|
Optionally accepts iterable of sessions to added on init
|
||||||
def __init__(self, old_sessions:Iterable=None):
|
Arguments: old_session: iterable of old UploadSession objects
|
||||||
#self._too_many: TooMany = None
|
"""
|
||||||
|
|
||||||
|
def __init__(self, old_sessions: List = None):
|
||||||
if old_sessions is None:
|
if old_sessions is None:
|
||||||
self.sessions = []
|
self.sessions = []
|
||||||
else:
|
else:
|
||||||
self.sessions = old_session
|
self.sessions = old_sessions
|
||||||
|
|
||||||
def add_session(self, session_or_block: Union(str, bytes, session.UploadSession))->session.UploadSession:
|
def add_session(self,
|
||||||
"""Create (or add existing) block upload session from a str/bytes block hex hash, existing UploadSession"""
|
session_or_block: Union[str,
|
||||||
if isinstance(session_or_block, session.UploadSession):
|
bytes,
|
||||||
if not session_or_block in self.sessions:
|
session.UploadSession
|
||||||
|
]
|
||||||
|
) -> session.UploadSession:
|
||||||
|
"""Create (or add existing) block upload session.
|
||||||
|
|
||||||
|
from a str/bytes block hex hash, existing UploadSession
|
||||||
|
"""
|
||||||
|
if isinstance(session_or_block, session.UploadSession):
|
||||||
|
if session_or_block not in self.sessions:
|
||||||
self.sessions.append(session_or_block)
|
self.sessions.append(session_or_block)
|
||||||
return session_or_block
|
return session_or_block
|
||||||
try:
|
try:
|
||||||
|
@ -50,46 +60,63 @@ class BlockUploadSessionManager:
|
||||||
except KeyError:
|
except KeyError:
|
||||||
pass
|
pass
|
||||||
# convert bytes hash to str
|
# convert bytes hash to str
|
||||||
if isinstance(session_or_block, bytes): session_or_block = bytesconverter.bytes_to_str(session_or_block)
|
if isinstance(session_or_block, bytes):
|
||||||
|
session_or_block = bytesconverter.bytes_to_str(session_or_block)
|
||||||
# intentionally not elif
|
# intentionally not elif
|
||||||
if isinstance(session_or_block, str):
|
if isinstance(session_or_block, str):
|
||||||
new_session = session.UploadSession(session_or_block)
|
new_session = session.UploadSession(session_or_block)
|
||||||
self.sessions.append(new_session)
|
self.sessions.append(new_session)
|
||||||
return new_session
|
return new_session
|
||||||
|
raise ValueError
|
||||||
|
|
||||||
def get_session(self, block_hash: Union(str, bytes))->session.UploadSession:
|
def get_session(self,
|
||||||
block_hash = reconstructhash.deconstruct_hash(bytesconverter.bytes_to_str(block_hash))
|
block_hash: Union[str, bytes]
|
||||||
for session in self.sessions:
|
) -> session.UploadSession:
|
||||||
if session.block_hash == block_hash: return session
|
block_hash = reconstructhash.deconstruct_hash(
|
||||||
|
bytesconverter.bytes_to_str(block_hash))
|
||||||
|
for sess in self.sessions:
|
||||||
|
if sess.block_hash == block_hash:
|
||||||
|
return sess
|
||||||
raise KeyError
|
raise KeyError
|
||||||
|
|
||||||
def clean_session(self, specific_session: Union[str, UploadSession]=None):
|
def clean_session(self,
|
||||||
comm_inst: OnionrCommunicatorDaemon = self._too_many.get_by_string("OnionrCommunicatorDaemon")
|
specific_session: Union[str, 'UploadSession'] = None):
|
||||||
sessions_to_delete = []
|
|
||||||
if comm_inst.getUptime() < 120: return
|
|
||||||
onlinePeerCount = len(comm_inst.onlinePeers)
|
|
||||||
|
|
||||||
# If we have no online peers right now,
|
|
||||||
if onlinePeerCount == 0: return
|
|
||||||
|
|
||||||
for session in self.sessions:
|
comm_inst: 'OnionrCommunicatorDaemon' # type: ignore
|
||||||
# if over 50% of peers that were online for a session have become unavailable, don't kill sessions
|
comm_inst = self._too_many.get_by_string( # pylint: disable=E1101 type: ignore
|
||||||
if session.total_success_count > onlinePeerCount:
|
"OnionrCommunicatorDaemon")
|
||||||
if onlinePeerCount / session.total_success_count >= 0.5: return
|
sessions_to_delete = []
|
||||||
|
if comm_inst.getUptime() < 120:
|
||||||
|
return
|
||||||
|
onlinePeerCount = len(comm_inst.onlinePeers)
|
||||||
|
|
||||||
|
# If we have no online peers right now,
|
||||||
|
if onlinePeerCount == 0:
|
||||||
|
return
|
||||||
|
|
||||||
|
for sess in self.sessions:
|
||||||
|
# if over 50% of peers that were online for a session have
|
||||||
|
# become unavailable, don't kill sessions
|
||||||
|
if sess.total_success_count > onlinePeerCount:
|
||||||
|
if onlinePeerCount / sess.total_success_count >= 0.5:
|
||||||
|
return
|
||||||
# Clean sessions if they have uploaded to enough online peers
|
# Clean sessions if they have uploaded to enough online peers
|
||||||
if session.total_success_count <= 0: continue
|
if sess.total_success_count <= 0:
|
||||||
if (session.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
|
continue
|
||||||
sessions_to_delete.append(session)
|
if (sess.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
|
||||||
for session in sessions_to_delete:
|
sessions_to_delete.append(sess)
|
||||||
|
for sess in sessions_to_delete:
|
||||||
self.sessions.remove(session)
|
self.sessions.remove(session)
|
||||||
# TODO cleanup to one round of search
|
# TODO cleanup to one round of search
|
||||||
# Remove the blocks from the sessions, upload list, and waitforshare list
|
# Remove the blocks from the sessions, upload list,
|
||||||
|
# and waitforshare list
|
||||||
try:
|
try:
|
||||||
comm_inst.blocksToUpload.remove(reconstructhash.reconstruct_hash(session.block_hash))
|
comm_inst.blocksToUpload.remove(
|
||||||
|
reconstructhash.reconstruct_hash(sess.block_hash))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
try:
|
try:
|
||||||
comm_inst.blocksToUpload.remove(session.block_hash)
|
comm_inst.blocksToUpload.remove(sess.block_hash)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
localcommand.local_command('waitforshare/{session.block_hash}')
|
localcommand.local_command('waitforshare/{session.block_hash}')
|
||||||
|
|
Loading…
Reference in New Issue