linting refactoring communicator(utils) and reduced TOCTOU issus with online peer picking
parent
9fbee668aa
commit
e5f3866f9e
|
@ -1,9 +1,13 @@
|
||||||
'''
|
"""Onionr - Private P2P Communication.
|
||||||
Onionr - Private P2P Communication
|
|
||||||
|
|
||||||
clear offline peer in a communicator instance
|
clear offline peer in a communicator instance
|
||||||
'''
|
"""
|
||||||
'''
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import logger
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from communicator import OnionrCommunicatorDaemon
|
||||||
|
"""
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
@ -16,14 +20,16 @@
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
You should have received a copy of the GNU General Public License
|
||||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
'''
|
"""
|
||||||
import logger
|
|
||||||
def clear_offline_peer(comm_inst):
|
|
||||||
'''Removes the longest offline peer to retry later'''
|
def clear_offline_peer(comm_inst: 'OnionrCommunicatorDaemon'):
|
||||||
|
"""Remove the longest offline peer to retry later."""
|
||||||
try:
|
try:
|
||||||
removed = comm_inst.offlinePeers.pop(0)
|
removed = comm_inst.offlinePeers.pop(0)
|
||||||
except IndexError:
|
except IndexError:
|
||||||
pass
|
pass
|
||||||
else:
|
else:
|
||||||
logger.debug('Removed ' + removed + ' from offline list, will try them again.')
|
logger.debug('Removed ' + removed +
|
||||||
comm_inst.decrementThreadCount('clear_offline_peer')
|
' from offline list, will try them again.')
|
||||||
|
comm_inst.decrementThreadCount('clear_offline_peer')
|
||||||
|
|
|
@ -1,11 +1,14 @@
|
||||||
"""
|
"""Onionr - Private P2P Communication.
|
||||||
Onionr - Private P2P Communication
|
|
||||||
|
|
||||||
get online peers in a communicator instance
|
get online peers in a communicator instance
|
||||||
"""
|
"""
|
||||||
import time
|
import time
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from etc import humanreadabletime
|
from etc import humanreadabletime
|
||||||
import logger
|
import logger
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from communicator import OnionrCommunicatorDaemon
|
||||||
"""
|
"""
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
|
@ -22,25 +25,25 @@ import logger
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
def get_online_peers(comm_inst):
|
def get_online_peers(comm_inst: 'OnionrCommunicatorDaemon'):
|
||||||
"""
|
"""Manage the comm_inst.onlinePeers attribute list.
|
||||||
Manages the comm_inst.onlinePeers attribute list,
|
|
||||||
]connects to more peers if we have none connected
|
Connect to more peers if we have none connected
|
||||||
"""
|
"""
|
||||||
config = comm_inst.config
|
config = comm_inst.config
|
||||||
if config.get('general.offline_mode', False):
|
if config.get('general.offline_mode', False):
|
||||||
comm_inst.decrementThreadCount('get_online_peers')
|
comm_inst.decrementThreadCount('get_online_peers')
|
||||||
return
|
return
|
||||||
logger.debug('Refreshing peer pool...')
|
logger.debug('Refreshing peer pool...')
|
||||||
maxPeers = int(config.get('peers.max_connect', 10))
|
max_peers = int(config.get('peers.max_connect', 10))
|
||||||
needed = maxPeers - len(comm_inst.onlinePeers)
|
needed = max_peers - len(comm_inst.onlinePeers)
|
||||||
|
|
||||||
last_seen = 'never'
|
last_seen = 'never'
|
||||||
if not isinstance(comm_inst.lastNodeSeen, type(None)):
|
if not isinstance(comm_inst.lastNodeSeen, type(None)):
|
||||||
last_seen = humanreadabletime.human_readable_time(
|
last_seen = humanreadabletime.human_readable_time(
|
||||||
comm_inst.lastNodeSeen)
|
comm_inst.lastNodeSeen)
|
||||||
|
|
||||||
for i in range(needed):
|
for _ in range(needed):
|
||||||
if len(comm_inst.onlinePeers) == 0:
|
if len(comm_inst.onlinePeers) == 0:
|
||||||
comm_inst.connectNewPeer(useBootstrap=True)
|
comm_inst.connectNewPeer(useBootstrap=True)
|
||||||
else:
|
else:
|
||||||
|
@ -50,8 +53,7 @@ def get_online_peers(comm_inst):
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
if len(comm_inst.onlinePeers) == 0:
|
if len(comm_inst.onlinePeers) == 0:
|
||||||
logger.debug
|
logger.debug('Couldn\'t connect to any peers.' +
|
||||||
('Couldn\'t connect to any peers.' +
|
|
||||||
f' Last node seen {last_seen} ago.')
|
f' Last node seen {last_seen} ago.')
|
||||||
else:
|
else:
|
||||||
comm_inst.lastNodeSeen = time.time()
|
comm_inst.lastNodeSeen = time.time()
|
||||||
|
|
|
@ -1,9 +1,12 @@
|
||||||
'''
|
"""
|
||||||
Onionr - Private P2P Communication
|
Onionr - Private P2P Communication.
|
||||||
|
|
||||||
pick online peers in a communicator instance
|
pick online peers in a communicator instance
|
||||||
'''
|
"""
|
||||||
'''
|
import secrets
|
||||||
|
|
||||||
|
import onionrexceptions
|
||||||
|
"""
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
@ -16,17 +19,22 @@
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License
|
You should have received a copy of the GNU General Public License
|
||||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
'''
|
"""
|
||||||
import secrets
|
|
||||||
|
|
||||||
def pick_online_peer(comm_inst):
|
def pick_online_peer(comm_inst):
|
||||||
'''randomly picks peer from pool without bias (using secrets module)'''
|
"""Randomly picks peer from pool without bias (using secrets module)."""
|
||||||
ret_data = ''
|
ret_data = ''
|
||||||
|
peer_length = len(comm_inst.onlinePeers)
|
||||||
|
if peer_length <= 0:
|
||||||
|
raise onionrexceptions.OnlinePeerNeeded
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
peer_length = len(comm_inst.onlinePeers)
|
peer_length = len(comm_inst.onlinePeers)
|
||||||
if peer_length <= 0:
|
|
||||||
break
|
|
||||||
try:
|
try:
|
||||||
# get a random online peer, securely. May get stuck in loop if network is lost or if all peers in pool magically disconnect at once
|
# Get a random online peer, securely.
|
||||||
|
# May get stuck in loop if network is lost or if all peers in pool magically disconnect at once
|
||||||
ret_data = comm_inst.onlinePeers[secrets.randbelow(peer_length)]
|
ret_data = comm_inst.onlinePeers[secrets.randbelow(peer_length)]
|
||||||
except IndexError:
|
except IndexError:
|
||||||
pass
|
pass
|
||||||
|
|
|
@ -25,6 +25,8 @@ from utils import gettransports
|
||||||
from netcontroller import NetController
|
from netcontroller import NetController
|
||||||
from communicator import onlinepeers
|
from communicator import onlinepeers
|
||||||
from coredb import keydb
|
from coredb import keydb
|
||||||
|
import onionrexceptions
|
||||||
|
|
||||||
def announce_node(daemon):
|
def announce_node(daemon):
|
||||||
'''Announce our node to our peers'''
|
'''Announce our node to our peers'''
|
||||||
ret_data = False
|
ret_data = False
|
||||||
|
@ -41,12 +43,17 @@ def announce_node(daemon):
|
||||||
peer = i
|
peer = i
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
peer = onlinepeers.pick_online_peer(daemon)
|
try:
|
||||||
|
peer = onlinepeers.pick_online_peer(daemon)
|
||||||
|
except onionrexceptions.OnlinePeerNeeded:
|
||||||
|
peer = ""
|
||||||
|
|
||||||
for x in range(1):
|
for _ in range(1):
|
||||||
try:
|
try:
|
||||||
ourID = gettransports.get()[0]
|
ourID = gettransports.get()[0]
|
||||||
except IndexError:
|
if not peer:
|
||||||
|
raise onionrexceptions.OnlinePeerNeeded
|
||||||
|
except (IndexError, onionrexceptions.OnlinePeerNeeded):
|
||||||
break
|
break
|
||||||
|
|
||||||
url = 'http://' + peer + '/announce'
|
url = 'http://' + peer + '/announce'
|
||||||
|
|
|
@ -43,8 +43,7 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
|
||||||
# Iterate the block queue in the communicator
|
# Iterate the block queue in the communicator
|
||||||
for blockHash in list(comm_inst.blockQueue):
|
for blockHash in list(comm_inst.blockQueue):
|
||||||
count += 1
|
count += 1
|
||||||
if len(comm_inst.onlinePeers) == 0:
|
|
||||||
break
|
|
||||||
triedQueuePeers = [] # List of peers we've tried for a block
|
triedQueuePeers = [] # List of peers we've tried for a block
|
||||||
try:
|
try:
|
||||||
blockPeers = list(comm_inst.blockQueue[blockHash])
|
blockPeers = list(comm_inst.blockQueue[blockHash])
|
||||||
|
@ -62,9 +61,15 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
|
||||||
if blockHash in comm_inst.currentDownloading:
|
if blockHash in comm_inst.currentDownloading:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
if len(comm_inst.onlinePeers) == 0:
|
||||||
|
break
|
||||||
|
|
||||||
comm_inst.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
|
comm_inst.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
|
||||||
if len(blockPeers) == 0:
|
if len(blockPeers) == 0:
|
||||||
peerUsed = onlinepeers.pick_online_peer(comm_inst)
|
try:
|
||||||
|
peerUsed = onlinepeers.pick_online_peer(comm_inst)
|
||||||
|
except onionrexceptions.OnlinePeerNeeded:
|
||||||
|
continue
|
||||||
else:
|
else:
|
||||||
blockPeers = onionrcrypto.cryptoutils.random_shuffle(blockPeers)
|
blockPeers = onionrcrypto.cryptoutils.random_shuffle(blockPeers)
|
||||||
peerUsed = blockPeers.pop(0)
|
peerUsed = blockPeers.pop(0)
|
||||||
|
|
|
@ -21,6 +21,7 @@ import logger
|
||||||
from onionrutils import stringvalidators
|
from onionrutils import stringvalidators
|
||||||
from communicator import peeraction, onlinepeers
|
from communicator import peeraction, onlinepeers
|
||||||
from utils import gettransports
|
from utils import gettransports
|
||||||
|
import onionrexceptions
|
||||||
def lookup_new_peer_transports_with_communicator(comm_inst):
|
def lookup_new_peer_transports_with_communicator(comm_inst):
|
||||||
logger.info('Looking up new addresses...')
|
logger.info('Looking up new addresses...')
|
||||||
tryAmount = 1
|
tryAmount = 1
|
||||||
|
@ -32,8 +33,11 @@ def lookup_new_peer_transports_with_communicator(comm_inst):
|
||||||
if len(newPeers) > 10000:
|
if len(newPeers) > 10000:
|
||||||
# Don't get new peers if we have too many queued up
|
# Don't get new peers if we have too many queued up
|
||||||
break
|
break
|
||||||
peer = onlinepeers.pick_online_peer(comm_inst)
|
try:
|
||||||
newAdders = peeraction.peer_action(comm_inst, peer, action='pex')
|
peer = onlinepeers.pick_online_peer(comm_inst)
|
||||||
|
newAdders = peeraction.peer_action(comm_inst, peer, action='pex')
|
||||||
|
except onionrexceptions.OnlinePeerNeeded:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
newPeers = newAdders.split(',')
|
newPeers = newAdders.split(',')
|
||||||
except AttributeError:
|
except AttributeError:
|
||||||
|
|
|
@ -17,12 +17,15 @@
|
||||||
You should have received a copy of the GNU General Public License
|
You should have received a copy of the GNU General Public License
|
||||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
'''
|
'''
|
||||||
|
from gevent import time
|
||||||
|
|
||||||
import logger, onionrproofs
|
import logger, onionrproofs
|
||||||
from onionrutils import stringvalidators, epoch
|
from onionrutils import stringvalidators, epoch
|
||||||
from communicator import peeraction, onlinepeers
|
from communicator import peeraction, onlinepeers
|
||||||
from coredb import blockmetadb
|
from coredb import blockmetadb
|
||||||
from utils import reconstructhash
|
from utils import reconstructhash
|
||||||
from onionrblocks import onionrblacklist
|
from onionrblocks import onionrblacklist
|
||||||
|
import onionrexceptions
|
||||||
blacklist = onionrblacklist.OnionrBlackList()
|
blacklist = onionrblacklist.OnionrBlackList()
|
||||||
def lookup_blocks_from_communicator(comm_inst):
|
def lookup_blocks_from_communicator(comm_inst):
|
||||||
logger.info('Looking up new blocks')
|
logger.info('Looking up new blocks')
|
||||||
|
@ -43,7 +46,12 @@ def lookup_blocks_from_communicator(comm_inst):
|
||||||
if comm_inst.storage_counter.is_full():
|
if comm_inst.storage_counter.is_full():
|
||||||
logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
|
logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
|
||||||
break
|
break
|
||||||
peer = onlinepeers.pick_online_peer(comm_inst) # select random online peer
|
try:
|
||||||
|
# select random online peer
|
||||||
|
peer = onlinepeers.pick_online_peer(comm_inst)
|
||||||
|
except onionrexceptions.OnlinePeerNeeded:
|
||||||
|
time.sleep(1)
|
||||||
|
continue
|
||||||
# if we've already tried all the online peers this time around, stop
|
# if we've already tried all the online peers this time around, stop
|
||||||
if peer in triedPeers:
|
if peer in triedPeers:
|
||||||
if len(comm_inst.onlinePeers) == len(triedPeers):
|
if len(comm_inst.onlinePeers) == len(triedPeers):
|
||||||
|
|
|
@ -45,8 +45,11 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
|
||||||
comm_inst.decrementThreadCount(TIMER_NAME)
|
comm_inst.decrementThreadCount(TIMER_NAME)
|
||||||
return
|
return
|
||||||
session = session_manager.add_session(bl)
|
session = session_manager.add_session(bl)
|
||||||
for i in range(min(len(comm_inst.onlinePeers), 6)):
|
for _ in range(min(len(comm_inst.onlinePeers), 6)):
|
||||||
peer = onlinepeers.pick_online_peer(comm_inst)
|
try:
|
||||||
|
peer = onlinepeers.pick_online_peer(comm_inst)
|
||||||
|
except onionrexceptions.OnlinePeerNeeded:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
session.peer_exists[peer]
|
session.peer_exists[peer]
|
||||||
continue
|
continue
|
||||||
|
|
Loading…
Reference in New Issue