renamed onionr dir and bugfixes/linting progress
This commit is contained in:
Kevin Froman
parent
2b996da17f
commit
720efe4fca
226 changed files with 179 additions and 142 deletions
90
src/communicatorutils/uploadblocks/__init__.py
Executable file
90
src/communicatorutils/uploadblocks/__init__.py
Executable file
|
|
@ -0,0 +1,90 @@
|
|||
'''
|
||||
Onionr - Private P2P Communication
|
||||
|
||||
Upload blocks in the upload queue to peers from the communicator
|
||||
'''
|
||||
from __future__ import annotations
|
||||
'''
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
from typing import Union, TYPE_CHECKING
|
||||
import logger
|
||||
from communicatorutils import proxypicker
|
||||
import onionrexceptions
|
||||
from onionrblocks import onionrblockapi as block
|
||||
from onionrutils import localcommand, stringvalidators, basicrequests
|
||||
from communicator import onlinepeers
|
||||
import onionrcrypto
|
||||
from . import sessionmanager
|
||||
|
||||
def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
|
||||
"""Accepts a communicator instance and uploads blocks from its upload queue"""
|
||||
"""when inserting a block, we try to upload
|
||||
it to a few peers to add some deniability & increase functionality"""
|
||||
TIMER_NAME = "upload_blocks_from_communicator"
|
||||
|
||||
session_manager: sessionmanager.BlockUploadSessionManager = comm_inst.shared_state.get(sessionmanager.BlockUploadSessionManager)
|
||||
triedPeers = []
|
||||
finishedUploads = []
|
||||
comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(comm_inst.blocksToUpload)
|
||||
if len(comm_inst.blocksToUpload) != 0:
|
||||
for bl in comm_inst.blocksToUpload:
|
||||
if not stringvalidators.validate_hash(bl):
|
||||
logger.warn('Requested to upload invalid block', terminal=True)
|
||||
comm_inst.decrementThreadCount(TIMER_NAME)
|
||||
return
|
||||
session = session_manager.add_session(bl)
|
||||
for i in range(min(len(comm_inst.onlinePeers), 6)):
|
||||
peer = onlinepeers.pick_online_peer(comm_inst)
|
||||
try:
|
||||
session.peer_exists[peer]
|
||||
continue
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
if session.peer_fails[peer] > 3: continue
|
||||
except KeyError:
|
||||
pass
|
||||
if peer in triedPeers: continue
|
||||
triedPeers.append(peer)
|
||||
url = f'http://{peer}/upload'
|
||||
try:
|
||||
data = block.Block(bl).getRaw()
|
||||
except onionrexceptions.NoDataAvailable:
|
||||
finishedUploads.append(bl)
|
||||
break
|
||||
proxyType = proxypicker.pick_proxy(peer)
|
||||
logger.info(f"Uploading block {bl[:8]} to {peer}", terminal=True)
|
||||
resp = basicrequests.do_post_request(url, data=data, proxyType=proxyType, content_type='application/octet-stream')
|
||||
if not resp == False:
|
||||
if resp == 'success':
|
||||
session.success()
|
||||
session.peer_exists[peer] = True
|
||||
elif resp == 'exists':
|
||||
session.success()
|
||||
session.peer_exists[peer] = True
|
||||
else:
|
||||
session.fail()
|
||||
session.fail_peer(peer)
|
||||
comm_inst.getPeerProfileInstance(peer).addScore(-5)
|
||||
logger.warn(f'Failed to upload {bl[:8]}, reason: {resp[:15]}', terminal=True)
|
||||
else:
|
||||
session.fail()
|
||||
session_manager.clean_session()
|
||||
for x in finishedUploads:
|
||||
try:
|
||||
comm_inst.blocksToUpload.remove(x)
|
||||
except ValueError:
|
||||
pass
|
||||
comm_inst.decrementThreadCount(TIMER_NAME)
|
||||
54
src/communicatorutils/uploadblocks/session.py
Normal file
54
src/communicatorutils/uploadblocks/session.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
"""
|
||||
Onionr - Private P2P Communication
|
||||
|
||||
Virtual upload "sessions" for blocks
|
||||
"""
|
||||
from __future__ import annotations
|
||||
"""
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
from typing import Union
|
||||
|
||||
from onionrutils import stringvalidators
|
||||
from onionrutils import bytesconverter
|
||||
from onionrutils import epoch
|
||||
from utils import reconstructhash
|
||||
|
||||
class UploadSession:
|
||||
"""Manages statistics for an Onionr block upload session
|
||||
|
||||
accepting a block hash (incl. unpadded) as an argument"""
|
||||
def __init__(self, block_hash: Union[str, bytes]):
|
||||
block_hash = bytesconverter.bytes_to_str(block_hash)
|
||||
block_hash = reconstructhash.reconstruct_hash(block_hash)
|
||||
if not stringvalidators.validate_hash(block_hash): raise ValueError
|
||||
|
||||
self.start_time = epoch.get_epoch()
|
||||
self.block_hash = reconstructhash.deconstruct_hash(block_hash)
|
||||
self.total_fail_count: int = 0
|
||||
self.total_success_count: int = 0
|
||||
self.peer_fails = {}
|
||||
self.peer_exists = {}
|
||||
|
||||
def fail_peer(self, peer):
|
||||
try:
|
||||
self.peer_fails[peer] += 1
|
||||
except KeyError:
|
||||
self.peer_fails[peer] = 0
|
||||
|
||||
def fail(self):
|
||||
self.total_fail_count += 1
|
||||
|
||||
def success(self):
|
||||
self.total_success_count += 1
|
||||
95
src/communicatorutils/uploadblocks/sessionmanager.py
Normal file
95
src/communicatorutils/uploadblocks/sessionmanager.py
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
"""
|
||||
Onionr - Private P2P Communication
|
||||
|
||||
Manager for upload 'sessions'
|
||||
"""
|
||||
from __future__ import annotations
|
||||
"""
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
from typing import Iterable, Union
|
||||
|
||||
from onionrutils import bytesconverter
|
||||
from onionrutils import localcommand
|
||||
from etc import onionrvalues
|
||||
from etc import waitforsetvar
|
||||
from utils import reconstructhash
|
||||
|
||||
from . import session
|
||||
|
||||
class BlockUploadSessionManager:
|
||||
"""Holds block UploadSession instances. Optionally accepts iterable of sessions to added on init
|
||||
|
||||
Arguments: old_session: iterable of old UploadSession objects"""
|
||||
def __init__(self, old_sessions:Iterable=None):
|
||||
#self._too_many: TooMany = None
|
||||
if old_sessions is None:
|
||||
self.sessions = []
|
||||
else:
|
||||
self.sessions = old_session
|
||||
|
||||
def add_session(self, session_or_block: Union(str, bytes, session.UploadSession))->session.UploadSession:
|
||||
"""Create (or add existing) block upload session from a str/bytes block hex hash, existing UploadSession"""
|
||||
if isinstance(session_or_block, session.UploadSession):
|
||||
if not session_or_block in self.sessions:
|
||||
self.sessions.append(session_or_block)
|
||||
return session_or_block
|
||||
try:
|
||||
return self.get_session(session_or_block)
|
||||
except KeyError:
|
||||
pass
|
||||
# convert bytes hash to str
|
||||
if isinstance(session_or_block, bytes): session_or_block = bytesconverter.bytes_to_str(session_or_block)
|
||||
# intentionally not elif
|
||||
if isinstance(session_or_block, str):
|
||||
new_session = session.UploadSession(session_or_block)
|
||||
self.sessions.append(new_session)
|
||||
return new_session
|
||||
|
||||
def get_session(self, block_hash: Union(str, bytes))->session.UploadSession:
|
||||
block_hash = reconstructhash.deconstruct_hash(bytesconverter.bytes_to_str(block_hash))
|
||||
for session in self.sessions:
|
||||
if session.block_hash == block_hash: return session
|
||||
raise KeyError
|
||||
|
||||
def clean_session(self, specific_session: Union[str, UploadSession]=None):
|
||||
comm_inst: OnionrCommunicatorDaemon = self._too_many.get_by_string("OnionrCommunicatorDaemon")
|
||||
sessions_to_delete = []
|
||||
if comm_inst.getUptime() < 120: return
|
||||
onlinePeerCount = len(comm_inst.onlinePeers)
|
||||
|
||||
# If we have no online peers right now,
|
||||
if onlinePeerCount == 0: return
|
||||
|
||||
for session in self.sessions:
|
||||
# if over 50% of peers that were online for a session have become unavailable, don't kill sessions
|
||||
if session.total_success_count > onlinePeerCount:
|
||||
if onlinePeerCount / session.total_success_count >= 0.5: return
|
||||
# Clean sessions if they have uploaded to enough online peers
|
||||
if session.total_success_count <= 0: continue
|
||||
if (session.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
|
||||
sessions_to_delete.append(session)
|
||||
for session in sessions_to_delete:
|
||||
self.sessions.remove(session)
|
||||
# TODO cleanup to one round of search
|
||||
# Remove the blocks from the sessions, upload list, and waitforshare list
|
||||
try:
|
||||
comm_inst.blocksToUpload.remove(reconstructhash.reconstruct_hash(session.block_hash))
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
comm_inst.blocksToUpload.remove(session.block_hash)
|
||||
except ValueError:
|
||||
pass
|
||||
localcommand.local_command('waitforshare/{session.block_hash}')
|
||||
Loading…
Add table
Add a link
Reference in a new issue