formatting and comment improvements, cleanup

src/communicatorutils/announcenode.py

@@ -1,14 +1,11 @@
 """
-    Onionr - Private P2P Communication.
+Onionr - Private P2P Communication.
 
 Use a communicator instance to announce
 our transport address to connected nodes
 """
-import base64
-import onionrproofs
 import logger
-from etc import onionrvalues
+from onionrutils import basicrequests
-from onionrutils import basicrequests, bytesconverter
 from utils import gettransports
 from netcontroller import NetController
 from communicator import onlinepeers
@@ -33,7 +30,6 @@ import onionrexceptions
 def announce_node(daemon):
     """Announce our node to our peers."""
     ret_data = False
-    announce_fail = False
 
     # Do not let announceCache get too large
     if len(daemon.announceCache) >= 10000:

src/communicatorutils/connectnewpeers.py

@@ -1,9 +1,14 @@
 """Onionr - Private P2P Communication.
 
-Connect a new peer to our communicator instance. Does so randomly if no peer is specified
+Connect a new peer to our communicator instance.
+Does so randomly if no peer is specified
 """
-import time, sys, secrets
+import time
-import onionrexceptions, logger, onionrpeers
+import secrets
+
+import onionrexceptions
+import logger
+import onionrpeers
 from utils import networkmerger, gettransports
 from onionrutils import stringvalidators, epoch
 from communicator import peeraction, bootstrappeers
@@ -33,14 +38,18 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
         if stringvalidators.validate_transport(peer):
             peerList = [peer]
         else:
-            raise onionrexceptions.InvalidAddress('Will not attempt connection test to invalid address')
+            raise onionrexceptions.InvalidAddress(
+                'Will not attempt connection test to invalid address')
     else:
         peerList = keydb.listkeys.list_adders()
 
     mainPeerList = keydb.listkeys.list_adders()
     peerList = onionrpeers.get_score_sorted_peer_list()
 
-    # If we don't have enough peers connected or random chance, select new peers to try
+    """
+    If we don't have enough peers connected or random chance,
+    select new peers to try
+    """
     if len(peerList) < 8 or secrets.randbelow(4) == 3:
         tryingNew = []
         for x in comm_inst.newPeers:
@@ -61,8 +70,12 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
         # Don't connect to our own address
         if address in transports:
             continue
-        # Don't connect to invalid address or if its already been tried/connected, or if its cooled down
+        """Don't connect to invalid address or
-        if len(address) == 0 or address in tried or address in comm_inst.onlinePeers or address in comm_inst.cooldownPeer:
+        if its already been tried/connected, or if its cooled down
+        """
+        if len(address) == 0 or address in tried \
+            or address in comm_inst.onlinePeers \
+                or address in comm_inst.cooldownPeer:
             continue
         if comm_inst.shutdown:
             return
@@ -71,7 +84,7 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
         if ret == 'pong!':
             time.sleep(0.1)
             if address not in mainPeerList:
-                # Add a peer to our list if it isn't already since it successfully connected
+                # Add a peer to our list if it isn't already since it connected
                 networkmerger.mergeAdders(address)
             if address not in comm_inst.onlinePeers:
                 logger.info('Connected to ' + address, terminal=True)
@@ -84,7 +97,8 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
                 if profile.address == address:
                     break
             else:
-                comm_inst.peerProfiles.append(onionrpeers.PeerProfiles(address))
+                comm_inst.peerProfiles.append(
+                    onionrpeers.PeerProfiles(address))
             break
         else:
             # Mark a peer as tried if they failed to respond to ping

src/communicatorutils/cooldownpeer.py

@@ -1,9 +1,10 @@
-'''
+"""Onionr - Private P2P Communication.
-    Onionr - Private P2P Communication
 
-    Select a random online peer in a communicator instance and have them "cool down"
+Select random online peer in a communicator instance and have them "cool down"
-'''
+"""
-'''
+from onionrutils import epoch
+from communicator import onlinepeers
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,39 +17,39 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
-from onionrutils import epoch
+
-from communicator import onlinepeers
+
 def cooldown_peer(comm_inst):
-    '''Randomly add an online peer to cooldown, so we can connect a new one'''
+    """Randomly add an online peer to cooldown, so we can connect a new one."""
     config = comm_inst.config
-    onlinePeerAmount = len(comm_inst.onlinePeers)
+    online_peer_amount = len(comm_inst.onlinePeers)
     minTime = 300
-    cooldownTime = 600
+    cooldown_time = 600
-    toCool = ''
+    to_cool = ''
     tempConnectTimes = dict(comm_inst.connectTimes)
 
     # Remove peers from cooldown that have been there long enough
     tempCooldown = dict(comm_inst.cooldownPeer)
     for peer in tempCooldown:
-        if (epoch.get_epoch() - tempCooldown[peer]) >= cooldownTime:
+        if (epoch.get_epoch() - tempCooldown[peer]) >= cooldown_time:
             del comm_inst.cooldownPeer[peer]
 
     # Cool down a peer, if we have max connections alive for long enough
-    if onlinePeerAmount >= config.get('peers.max_connect', 10, save = True):
+    if online_peer_amount >= config.get('peers.max_connect', 10, save=True):
         finding = True
 
         while finding:
             try:
-                toCool = min(tempConnectTimes, key=tempConnectTimes.get)
+                to_cool = min(tempConnectTimes, key=tempConnectTimes.get)
-                if (epoch.get_epoch() - tempConnectTimes[toCool]) < minTime:
+                if (epoch.get_epoch() - tempConnectTimes[to_cool]) < minTime:
-                    del tempConnectTimes[toCool]
+                    del tempConnectTimes[to_cool]
                 else:
                     finding = False
             except ValueError:
                 break
         else:
-            onlinepeers.remove_online_peer(comm_inst, toCool)
+            onlinepeers.remove_online_peer(comm_inst, to_cool)
-            comm_inst.cooldownPeer[toCool] = epoch.get_epoch()
+            comm_inst.cooldownPeer[to_cool] = epoch.get_epoch()
 
-    comm_inst.decrementThreadCount('cooldown_peer')
+    comm_inst.decrementThreadCount('cooldown_peer')

src/communicatorutils/deniableinserts.py

@@ -1,9 +1,12 @@
-'''
+"""Onionr - Private P2P Communication.
-    Onionr - Private P2P Communication
 
-    Use the communicator to insert fake mail messages
+Use the communicator to insert fake mail messages
-'''
+"""
-'''
+import secrets
+
+from etc import onionrvalues
+import onionrblocks
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,17 +19,18 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
-import secrets
+
-from etc import onionrvalues
+
-import onionrblocks
 def insert_deniable_block(comm_inst):
-    '''Insert a fake block in order to make it more difficult to track real blocks'''
+    """Insert a fake block to make it more difficult to track real blocks."""
     fakePeer = ''
     chance = 10
     if secrets.randbelow(chance) == (chance - 1):
         # This assumes on the libsodium primitives to have key-privacy
         fakePeer = onionrvalues.DENIABLE_PEER_ADDRESS
         data = secrets.token_hex(secrets.randbelow(5120) + 1)
-        onionrblocks.insert(data, header='pm', encryptType='asym', asymPeer=fakePeer, disableForward=True, meta={'subject': 'foo'})
+        onionrblocks.insert(data, header='pm', encryptType='asym',
-    comm_inst.decrementThreadCount('insert_deniable_block')
+                            asymPeer=fakePeer, disableForward=True,
+                            meta={'subject': 'foo'})
+    comm_inst.decrementThreadCount('insert_deniable_block')

src/communicatorutils/housekeeping.py

@@ -4,6 +4,7 @@ Cleanup old Onionr blocks and forward secrecy keys using the communicator.
 Ran from a communicator timer usually
 """
 import sqlite3
+
 import logger
 from onionrusers import onionrusers
 from onionrutils import epoch
@@ -67,7 +68,8 @@ def clean_keys(comm_inst):
     time = epoch.get_epoch()
     deleteKeys = []
 
-    for entry in c.execute("SELECT * FROM forwardKeys WHERE expire <= ?", (time,)):
+    for entry in c.execute(
+            "SELECT * FROM forwardKeys WHERE expire <= ?", (time,)):
         logger.debug('Forward key: %s' % entry[1])
         deleteKeys.append(entry[1])
 

src/communicatorutils/lookupadders.py

@@ -1,9 +1,14 @@
-'''
+"""
-    Onionr - Private P2P Communication
+Onionr - Private P2P Communication.
 
-    Lookup new peer transport addresses using the communicator
+Lookup new peer transport addresses using the communicator
-'''
+"""
-'''
+import logger
+from onionrutils import stringvalidators
+from communicator import peeraction, onlinepeers
+from utils import gettransports
+import onionrexceptions
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,12 +21,9 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
-import logger
+
-from onionrutils import stringvalidators
+
-from communicator import peeraction, onlinepeers
-from utils import gettransports
-import onionrexceptions
 def lookup_new_peer_transports_with_communicator(comm_inst):
     logger.info('Looking up new addresses...')
     tryAmount = 1
@@ -47,7 +49,8 @@ def lookup_new_peer_transports_with_communicator(comm_inst):
         invalid = []
         for x in newPeers:
             x = x.strip()
-            if not stringvalidators.validate_transport(x) or x in comm_inst.newPeers or x in transports:
+            if not stringvalidators.validate_transport(x) \
+                    or x in comm_inst.newPeers or x in transports:
                 # avoid adding if its our address
                 invalid.append(x)
         for x in invalid:
@@ -56,4 +59,5 @@ def lookup_new_peer_transports_with_communicator(comm_inst):
             except ValueError:
                 pass
         comm_inst.newPeers.extend(newPeers)
-    comm_inst.decrementThreadCount('lookup_new_peer_transports_with_communicator')
+    comm_inst.decrementThreadCount(
+        'lookup_new_peer_transports_with_communicator')

src/communicatorutils/netcheck.py

@@ -1,9 +1,9 @@
 """
-    Onionr - Private P2P Communication
+Onionr - Private P2P Communication.
 
-    Determine if our node is able to use Tor based
+Determine if our node is able to use Tor based
-    on the status of a communicator instance
+on the status of a communicator instance
-    and the result of pinging onion http servers
+and the result of pinging onion http servers
 """
 import logger
 from utils import netutils
@@ -26,8 +26,10 @@ from . import restarttor
 
 
 def net_check(comm_inst):
-    """Check if we are connected to the internet
+    """Check if we are connected to the internet.
-    or not when we can't connect to any peers"""
+
+    or not when we can't connect to any peers
+    """
     # for detecting if we have received incoming connections recently
     rec = False
     if len(comm_inst.onlinePeers) == 0:
@@ -42,8 +44,8 @@ def net_check(comm_inst):
             if not comm_inst.shutdown:
                 if not comm_inst.config.get('general.offline_mode', False):
                     logger.warn('Network check failed, are you connected to ' +
-                                'the Internet, and is Tor working? ' + 
+                                'the Internet, and is Tor working? ' +
-                                'This is usually temporary, but bugs and censorship can cause this to persist, in which case you should report it to beardog [at] mailbox.org',
+                                'This is usually temporary, but bugs and censorship can cause this to persist, in which case you should report it to beardog [at] mailbox.org',  # noqa
                                 terminal=True)
                     restarttor.restart(comm_inst)
                     comm_inst.offlinePeers = []

src/communicatorutils/onionrcommunicatortimers.py

@@ -1,10 +1,20 @@
-'''
+"""
-    Onionr - Private P2P Communication
+Onionr - Private P2P Communication.
 
-    This file contains timer control for the communicator
+This file contains timer control for the communicator
-'''
+"""
 from __future__ import annotations  # thank you python, very cool
-'''
+import uuid
+import threading
+
+import onionrexceptions
+import logger
+
+from typing import TYPE_CHECKING
+from typing import Callable, NewType, Iterable
+if TYPE_CHECKING:
+    from communicator import OnionrCommunicatorDaemon
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -17,26 +27,18 @@ from __future__ import annotations  # thank you python, very cool
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
 
-import uuid
-import threading
-
-import onionrexceptions, logger
-
-from typing import TYPE_CHECKING
-from typing import Callable, NewType, Iterable
-from psutil import Process
-if TYPE_CHECKING:
-    from communicator import OnionrCommunicatorDaemon
 
 CallFreqSeconds = NewType('CallFreqSeconds', int)
 
+
 class OnionrCommunicatorTimers:
     def __init__(self, daemon_inst: OnionrCommunicatorDaemon,
-                timer_function: Callable, frequency: CallFreqSeconds,
+                 timer_function: Callable, frequency: CallFreqSeconds,
-                make_thread:bool=True, thread_amount:int=1, max_threads:int=5,
+                 make_thread: bool = True,
-                requires_peer:bool=False, my_args:Iterable=[]):
+                 thread_amount: int = 1, max_threads: int = 5,
+                 requires_peer: bool = False, my_args: Iterable = []):
         self.timer_function = timer_function
         self.frequency = frequency
         self.thread_amount = thread_amount
@@ -51,30 +53,44 @@ class OnionrCommunicatorTimers:
 
     def processTimer(self):
 
-        # mark how many instances of a thread we have (decremented at thread end)
+        # mark # of instances of a thread we have (decremented at thread end)
         try:
             self.daemon_inst.threadCounts[self.timer_function.__name__]
         except KeyError:
             self.daemon_inst.threadCounts[self.timer_function.__name__] = 0
 
-        # execute thread if it is time, and we are not missing *required* online peer
+        # execute timer's func, if we are not missing *required* online peer
         if self.count == self.frequency and not self.daemon_inst.shutdown:
             try:
-                if self.requires_peer and len(self.daemon_inst.onlinePeers) == 0:
+                if self.requires_peer and \
+                        len(self.daemon_inst.onlinePeers) == 0:
                     raise onionrexceptions.OnlinePeerNeeded
             except onionrexceptions.OnlinePeerNeeded:
                 return
             else:
                 if self.make_thread:
                     for i in range(self.thread_amount):
-                        if self.daemon_inst.threadCounts[self.timer_function.__name__] >= self.max_threads:
+                        """
-                            logger.debug('%s is currently using the maximum number of threads, not starting another.' % self.timer_function.__name__)
+                        Log if a timer has max num of active threads
+                        If this logs frequently it is indicative of a bug
+                        or need for optimization
+                        """
+                        if self.daemon_inst.threadCounts[
+                                self.timer_function.__name__] >= \
+                                self.max_threads:
+                            logger.debug(
+                                f'{self.timer_function.__name__} is currently using the maximum number of threads, not starting another.')  # noqa
+                        # if active number of threads for timer not reached yet
                         else:
-                            self.daemon_inst.threadCounts[self.timer_function.__name__] += 1
+                            self.daemon_inst.threadCounts[
-                            newThread = threading.Thread(target=self.timer_function, args=self.args, daemon=True,
+                                self.timer_function.__name__] += 1
-                                                        name=self.timer_function.__name__ + ' - ' + str(uuid.uuid4()))
+                            newThread = threading.Thread(
+                                target=self.timer_function, args=self.args,
+                                daemon=True,
+                                name=self.timer_function.__name__ + ' - ' +
+                                str(uuid.uuid4()))
                             newThread.start()
                 else:
                     self.timer_function()
-            self.count = -1 # negative 1 because its incremented at bottom
+            self.count = -1  # negative 1 because its incremented at bottom
         self.count += 1

src/communicatorutils/proxypicker.py

@@ -1,9 +1,9 @@
-'''
+"""
-    Onionr - Private P2P Communication
+Onionr - Private P2P Communication.
 
-    Just picks a proxy to use based on a peer's address
+Pick a proxy to use based on a peer's address
-'''
+"""
-'''
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,11 +16,13 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
+
 
 def pick_proxy(peer_address):
     if peer_address.endswith('.onion'):
         return 'tor'
     elif peer_address.endswith('.i2p'):
         return 'i2p'
-    raise ValueError(f"Peer address was not string ending with acceptable value: {peer_address}")
+    raise ValueError(
+        f"Peer address not ending with acceptable domain: {peer_address}")

src/communicatorutils/restarttor.py

@@ -1,5 +1,25 @@
+"""
+Onionr - Private P2P Communication.
+
+Restart Onionr managed Tor
+"""
 import netcontroller
 import config
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
 
 def restart(comm_inst):
     if not config.get('tor.use_existing_tor', False):

src/communicatorutils/servicecreator.py

@@ -1,9 +1,14 @@
-'''
+"""
-    Onionr - Private P2P Communication
+Onionr - Private P2P Communication.
 
-    Creates an onionr direct connection service by scanning all connection blocks
+Creates an onionr direct connection service by scanning all connection blocks
-'''
+"""
-'''
+import communicator
+from onionrblocks import onionrblockapi
+from onionrutils import stringvalidators, bytesconverter
+from coredb import blockmetadb
+from onionrservices import server_exists
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,26 +21,23 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
-import communicator
+
-from onionrblocks import onionrblockapi
+
-import logger
-from onionrutils import stringvalidators, bytesconverter
-from coredb import blockmetadb
-from onionrservices import server_exists
 def service_creator(daemon):
     assert isinstance(daemon, communicator.OnionrCommunicatorDaemon)
-    
+
     # Find socket connection blocks
     # TODO cache blocks and only look at recently received ones
     con_blocks = blockmetadb.get_blocks_by_type('con')
     for b in con_blocks:
-        if not b in daemon.active_services:
+        if b not in daemon.active_services:
             bl = onionrblockapi.Block(b, decrypt=True)
             bs = bytesconverter.bytes_to_str(bl.bcontent) + '.onion'
             if server_exists(bl.signer):
                 continue
-            if stringvalidators.validate_pub_key(bl.signer) and stringvalidators.validate_transport(bs):
+            if stringvalidators.validate_pub_key(bl.signer) and \
+                    stringvalidators.validate_transport(bs):
                 signer = bytesconverter.bytes_to_str(bl.signer)
                 daemon.active_services.append(b)
                 daemon.active_services.append(signer)

src/onionrblocks/blocklist.py

@@ -1,3 +1,7 @@
+"""Onionr - Private P2P Communication.
+
+Get an auto updating list of blocks
+"""
 from threading import Thread
 
 from watchdog.observers import Observer
@@ -7,6 +11,20 @@ from utils.identifyhome import identify_home
 from coredb.dbfiles import block_meta_db
 from coredb.blockmetadb import get_block_list, get_blocks_by_type
 from onionrutils.epoch import get_epoch
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
 
 class BlockList:
     def __init__(self, auto_refresh=True, block_type=''):

src/onionrblocks/insert/main.py

@@ -40,7 +40,9 @@ from onionrtypes import UserIDSecretKey
 
 
 def _check_upload_queue():
-    """Returns the current upload queue len
+    """
+    Return the current upload queue len.
+
     raises OverflowError if max, false if api not running
     """
     max_upload_queue: int = 5000
@@ -63,11 +65,11 @@ def insert_block(data: Union[str, bytes], header: str = 'txt',
                  expire: Union[int, None] = None, disableForward: bool = False,
                  signing_key: UserIDSecretKey = '') -> Union[str, bool]:
     """
-        Create and insert a block into the network.
+    Create and insert a block into the network.
 
-        encryptType must be specified to encrypt a block
+    encryptType must be specified to encrypt a block
-        if expire is less than date, assumes seconds into future.
+    if expire is less than date, assumes seconds into future.
-            if not assume exact epoch
+        if not assume exact epoch
     """
     our_private_key = crypto.priv_key
     our_pub_key = crypto.pub_key