dxs/onionr
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

minor bug fixes and linting improvements

Browse source
This commit is contained in:
Kevin Froman 2019-12-23 01:51:24 -06:00
parent 9329b07e3b
commit 87ea8d137b
8 changed files with 223 additions and 154 deletions
Download patch file Download diff file Expand all files Collapse all files

74
src/communicatorutils/uploadblocks/__init__.py
View file

@ -1,10 +1,22 @@
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
Upload blocks in the upload queue to peers from the communicator
'''
from __future__ import annotations
'''
Upload blocks in the upload queue to peers from the communicator
"""
from typing import TYPE_CHECKING
from . import sessionmanager
from onionrtypes import UserID
import logger
from communicatorutils import proxypicker
import onionrexceptions
from onionrblocks import onionrblockapi as block
from onionrutils import stringvalidators, basicrequests
import onionrcrypto
from communicator import onlinepeers
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
@ -17,28 +29,23 @@ from __future__ import annotations
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
from typing import Union, TYPE_CHECKING
import logger
from communicatorutils import proxypicker
import onionrexceptions
from onionrblocks import onionrblockapi as block
from onionrutils import localcommand, stringvalidators, basicrequests
from communicator import onlinepeers
import onionrcrypto
from . import sessionmanager
from . import mixmate
"""
def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
"""Accepts a communicator instance and uploads blocks from its upload queue"""
def upload_blocks_from_communicator(comm_inst: 'OnionrCommunicatorDaemon'):
"""Accept a communicator instance + upload blocks from its upload queue."""
"""when inserting a block, we try to upload
it to a few peers to add some deniability & increase functionality"""
TIMER_NAME = "upload_blocks_from_communicator"
session_manager: sessionmanager.BlockUploadSessionManager = comm_inst.shared_state.get(sessionmanager.BlockUploadSessionManager)
triedPeers = []
session_manager: sessionmanager.BlockUploadSessionManager
session_manager = comm_inst.shared_state.get(
sessionmanager.BlockUploadSessionManager)
tried_peers: UserID = []
finishedUploads = []
comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(comm_inst.blocksToUpload)
comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(
comm_inst.blocksToUpload)
if len(comm_inst.blocksToUpload) != 0:
for bl in comm_inst.blocksToUpload:
if not stringvalidators.validate_hash(bl):
@ -57,21 +64,26 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
except KeyError:
pass
try:
if session.peer_fails[peer] > 3: continue
if session.peer_fails[peer] > 3:
continue
except KeyError:
pass
if peer in triedPeers: continue
triedPeers.append(peer)
if peer in tried_peers:
continue
tried_peers.append(peer)
url = f'http://{peer}/upload'
try:
data = block.Block(bl).getRaw()
except onionrexceptions.NoDataAvailable:
finishedUploads.append(bl)
break
proxyType = proxypicker.pick_proxy(peer)
logger.info(f"Uploading block {bl[:8]} to {peer}", terminal=True)
resp = basicrequests.do_post_request(url, data=data, proxyType=proxyType, content_type='application/octet-stream')
if not resp == False:
proxy_type = proxypicker.pick_proxy(peer)
logger.info(
f"Uploading block {bl[:8]} to {peer}", terminal=True)
resp = basicrequests.do_post_request(
url, data=data, proxyType=proxy_type,
content_type='application/octet-stream')
if resp is not False:
if resp == 'success':
session.success()
session.peer_exists[peer] = True
@ -82,7 +94,9 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
session.fail()
session.fail_peer(peer)
comm_inst.getPeerProfileInstance(peer).addScore(-5)
logger.warn(f'Failed to upload {bl[:8]}, reason: {resp[:15]}', terminal=True)
logger.warn(
f'Failed to upload {bl[:8]}, reason: {resp[:15]}',
terminal=True)
else:
session.fail()
session_manager.clean_session()

3
src/communicatorutils/uploadblocks/mixmate/pool.py
View file

@ -62,7 +62,8 @@ class UploadPool:
"""Get the hash pool in secure random order."""
if len(self._pool) != self._pool_size:
raise PoolNotReady
final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(list(self._pool))
final_pool: List[onionrtypes.BlockHash] = cryptoutils.random_shuffle(
list(self._pool))
self._pool.clear()
self.birthday = onionrutils.epoch.get_epoch()

7
src/communicatorutils/uploadblocks/session.py
View file

@ -2,8 +2,9 @@
Virtual upload "sessions" for blocks
"""
from typing import Union
from typing import Union, Dict
from onionrtypes import UserID
from onionrutils import stringvalidators
from onionrutils import bytesconverter
from onionrutils import epoch
@ -40,8 +41,8 @@ class UploadSession:
self.block_hash = reconstructhash.deconstruct_hash(block_hash)
self.total_fail_count: int = 0
self.total_success_count: int = 0
self.peer_fails = {}
self.peer_exists = {}
self.peer_fails: Dict[UserID, int] = {}
self.peer_exists: Dict[UserID, bool] = {}
def fail_peer(self, peer):
try:

119
src/communicatorutils/uploadblocks/sessionmanager.py
View file

@ -1,9 +1,17 @@
"""
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
Manager for upload 'sessions'
Manager for upload 'sessions'
"""
from __future__ import annotations
from typing import List, Union, TYPE_CHECKING
if TYPE_CHECKING:
from session import UploadSession
from onionrutils import bytesconverter
from onionrutils import localcommand
from etc import onionrvalues
from utils import reconstructhash
from . import session
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -18,31 +26,33 @@ from __future__ import annotations
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
from typing import Iterable, Union
from onionrutils import bytesconverter
from onionrutils import localcommand
from etc import onionrvalues
from etc import waitforsetvar
from utils import reconstructhash
from . import session
class BlockUploadSessionManager:
"""Holds block UploadSession instances. Optionally accepts iterable of sessions to added on init
"""Holds block UploadSession instances.
Arguments: old_session: iterable of old UploadSession objects"""
def __init__(self, old_sessions:Iterable=None):
#self._too_many: TooMany = None
Optionally accepts iterable of sessions to added on init
Arguments: old_session: iterable of old UploadSession objects
"""
def __init__(self, old_sessions: List = None):
if old_sessions is None:
self.sessions = []
else:
self.sessions = old_session
def add_session(self, session_or_block: Union(str, bytes, session.UploadSession))->session.UploadSession:
"""Create (or add existing) block upload session from a str/bytes block hex hash, existing UploadSession"""
if isinstance(session_or_block, session.UploadSession):
if not session_or_block in self.sessions:
self.sessions = old_sessions
def add_session(self,
session_or_block: Union[str,
bytes,
session.UploadSession
]
) -> session.UploadSession:
"""Create (or add existing) block upload session.
from a str/bytes block hex hash, existing UploadSession
"""
if isinstance(session_or_block, session.UploadSession):
if session_or_block not in self.sessions:
self.sessions.append(session_or_block)
return session_or_block
try:
@ -50,46 +60,63 @@ class BlockUploadSessionManager:
except KeyError:
pass
# convert bytes hash to str
if isinstance(session_or_block, bytes): session_or_block = bytesconverter.bytes_to_str(session_or_block)
if isinstance(session_or_block, bytes):
session_or_block = bytesconverter.bytes_to_str(session_or_block)
# intentionally not elif
if isinstance(session_or_block, str):
new_session = session.UploadSession(session_or_block)
self.sessions.append(new_session)
return new_session
raise ValueError
def get_session(self, block_hash: Union(str, bytes))->session.UploadSession:
block_hash = reconstructhash.deconstruct_hash(bytesconverter.bytes_to_str(block_hash))
for session in self.sessions:
if session.block_hash == block_hash: return session
def get_session(self,
block_hash: Union[str, bytes]
) -> session.UploadSession:
block_hash = reconstructhash.deconstruct_hash(
bytesconverter.bytes_to_str(block_hash))
for sess in self.sessions:
if sess.block_hash == block_hash:
return sess
raise KeyError
def clean_session(self, specific_session: Union[str, UploadSession]=None):
comm_inst: OnionrCommunicatorDaemon = self._too_many.get_by_string("OnionrCommunicatorDaemon")
sessions_to_delete = []
if comm_inst.getUptime() < 120: return
onlinePeerCount = len(comm_inst.onlinePeers)
# If we have no online peers right now,
if onlinePeerCount == 0: return
def clean_session(self,
specific_session: Union[str, 'UploadSession'] = None):
for session in self.sessions:
# if over 50% of peers that were online for a session have become unavailable, don't kill sessions
if session.total_success_count > onlinePeerCount:
if onlinePeerCount / session.total_success_count >= 0.5: return
comm_inst: 'OnionrCommunicatorDaemon' # type: ignore
comm_inst = self._too_many.get_by_string( # pylint: disable=E1101 type: ignore
"OnionrCommunicatorDaemon")
sessions_to_delete = []
if comm_inst.getUptime() < 120:
return
onlinePeerCount = len(comm_inst.onlinePeers)
# If we have no online peers right now,
if onlinePeerCount == 0:
return
for sess in self.sessions:
# if over 50% of peers that were online for a session have
# become unavailable, don't kill sessions
if sess.total_success_count > onlinePeerCount:
if onlinePeerCount / sess.total_success_count >= 0.5:
return
# Clean sessions if they have uploaded to enough online peers
if session.total_success_count <= 0: continue
if (session.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
sessions_to_delete.append(session)
for session in sessions_to_delete:
if sess.total_success_count <= 0:
continue
if (sess.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
sessions_to_delete.append(sess)
for sess in sessions_to_delete:
self.sessions.remove(session)
# TODO cleanup to one round of search
# Remove the blocks from the sessions, upload list, and waitforshare list
# Remove the blocks from the sessions, upload list,
# and waitforshare list
try:
comm_inst.blocksToUpload.remove(reconstructhash.reconstruct_hash(session.block_hash))
comm_inst.blocksToUpload.remove(
reconstructhash.reconstruct_hash(sess.block_hash))
except ValueError:
pass
try:
comm_inst.blocksToUpload.remove(session.block_hash)
comm_inst.blocksToUpload.remove(sess.block_hash)
except ValueError:
pass
localcommand.local_command('waitforshare/{session.block_hash}')