linting refactoring communicator(utils) and reduced TOCTOU issus with online peer picking

src/communicatorutils/announcenode.py

@@ -25,6 +25,8 @@ from utils import gettransports
 from netcontroller import NetController
 from communicator import onlinepeers
 from coredb import keydb
+import onionrexceptions
+
 def announce_node(daemon):
     '''Announce our node to our peers'''
     ret_data = False
@@ -41,12 +43,17 @@ def announce_node(daemon):
                 peer = i
                 break
         else:
-            peer = onlinepeers.pick_online_peer(daemon)
+            try:
+                peer = onlinepeers.pick_online_peer(daemon)
+            except onionrexceptions.OnlinePeerNeeded:
+                peer = ""
 
-        for x in range(1):
+        for _ in range(1):
             try:
                 ourID = gettransports.get()[0]
-            except IndexError:
+                if not peer:
+                    raise onionrexceptions.OnlinePeerNeeded
+            except (IndexError, onionrexceptions.OnlinePeerNeeded):
                 break
 
             url = 'http://' + peer + '/announce'

src/communicatorutils/downloadblocks/__init__.py

@@ -43,8 +43,7 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
     # Iterate the block queue in the communicator
     for blockHash in list(comm_inst.blockQueue):
         count += 1
-        if len(comm_inst.onlinePeers) == 0:
-            break
+
         triedQueuePeers = [] # List of peers we've tried for a block
         try:
             blockPeers = list(comm_inst.blockQueue[blockHash])
@@ -62,9 +61,15 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
         if blockHash in comm_inst.currentDownloading:
             continue
 
+        if len(comm_inst.onlinePeers) == 0:
+            break
+
         comm_inst.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
         if len(blockPeers) == 0:
-            peerUsed = onlinepeers.pick_online_peer(comm_inst)
+            try:
+                peerUsed = onlinepeers.pick_online_peer(comm_inst)
+            except onionrexceptions.OnlinePeerNeeded:
+                continue
         else:
             blockPeers = onionrcrypto.cryptoutils.random_shuffle(blockPeers)
             peerUsed = blockPeers.pop(0)

src/communicatorutils/lookupadders.py

@@ -21,6 +21,7 @@ import logger
 from onionrutils import stringvalidators
 from communicator import peeraction, onlinepeers
 from utils import gettransports
+import onionrexceptions
 def lookup_new_peer_transports_with_communicator(comm_inst):
     logger.info('Looking up new addresses...')
     tryAmount = 1
@@ -32,8 +33,11 @@ def lookup_new_peer_transports_with_communicator(comm_inst):
         if len(newPeers) > 10000:
             # Don't get new peers if we have too many queued up
             break
-        peer = onlinepeers.pick_online_peer(comm_inst)
-        newAdders = peeraction.peer_action(comm_inst, peer, action='pex')
+        try:
+            peer = onlinepeers.pick_online_peer(comm_inst)
+            newAdders = peeraction.peer_action(comm_inst, peer, action='pex')
+        except onionrexceptions.OnlinePeerNeeded:
+            continue
         try:
             newPeers = newAdders.split(',')
         except AttributeError:

src/communicatorutils/lookupblocks.py

@@ -17,12 +17,15 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
+from gevent import time
+
 import logger, onionrproofs
 from onionrutils import stringvalidators, epoch
 from communicator import peeraction, onlinepeers
 from coredb import blockmetadb
 from utils import reconstructhash
 from onionrblocks import onionrblacklist
+import onionrexceptions
 blacklist = onionrblacklist.OnionrBlackList()
 def lookup_blocks_from_communicator(comm_inst):
     logger.info('Looking up new blocks')
@@ -43,7 +46,12 @@ def lookup_blocks_from_communicator(comm_inst):
         if comm_inst.storage_counter.is_full():
             logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
             break
-        peer = onlinepeers.pick_online_peer(comm_inst) # select random online peer
+        try:
+            # select random online peer
+            peer = onlinepeers.pick_online_peer(comm_inst)
+        except onionrexceptions.OnlinePeerNeeded:
+            time.sleep(1)
+            continue
         # if we've already tried all the online peers this time around, stop
         if peer in triedPeers:
             if len(comm_inst.onlinePeers) == len(triedPeers):

src/communicatorutils/uploadblocks/__init__.py

@@ -45,8 +45,11 @@ def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
                 comm_inst.decrementThreadCount(TIMER_NAME)
                 return
             session = session_manager.add_session(bl)
-            for i in range(min(len(comm_inst.onlinePeers), 6)):
-                peer = onlinepeers.pick_online_peer(comm_inst)
+            for _ in range(min(len(comm_inst.onlinePeers), 6)):
+                try:
+                    peer = onlinepeers.pick_online_peer(comm_inst)
+                except onionrexceptions.OnlinePeerNeeded:
+                    continue
                 try:
                     session.peer_exists[peer]
                     continue