renamed onionr dir and bugfixes/linting progress

src/communicatorutils/downloadblocks/__init__.py

@@ -0,0 +1,133 @@
+'''
+    Onionr - Private P2P Communication
+
+    Download blocks using the communicator instance
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import communicator, onionrexceptions
+import logger, onionrpeers
+from onionrutils import blockmetadata, stringvalidators, validatemetadata
+from coredb import blockmetadb
+from . import shoulddownload
+from communicator import peeraction, onlinepeers
+import onionrcrypto, onionrstorage
+from onionrblocks import onionrblacklist, storagecounter
+def download_blocks_from_communicator(comm_inst):
+    '''Use Onionr communicator instance to download blocks in the communicator's queue'''
+    assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon)
+    blacklist = onionrblacklist.OnionrBlackList()
+    storage_counter = storagecounter.StorageCounter()
+    LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
+    count: int = 0
+    metadata_validation_result: bool = False
+    # Iterate the block queue in the communicator
+    for blockHash in list(comm_inst.blockQueue):
+        count += 1
+        if len(comm_inst.onlinePeers) == 0:
+            break
+        triedQueuePeers = [] # List of peers we've tried for a block
+        try:
+            blockPeers = list(comm_inst.blockQueue[blockHash])
+        except KeyError:
+            blockPeers = []
+        removeFromQueue = True
+
+        if not shoulddownload.should_download(comm_inst, blockHash):
+            continue
+
+        if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.is_full():
+            # Exit loop if shutting down or offline, or disk allocation reached
+            break
+        # Do not download blocks being downloaded
+        if blockHash in comm_inst.currentDownloading:
+            #logger.debug('Already downloading block %s...' % blockHash)
+            continue
+
+        comm_inst.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
+        if len(blockPeers) == 0:
+            peerUsed = onlinepeers.pick_online_peer(comm_inst)
+        else:
+            blockPeers = onionrcrypto.cryptoutils.random_shuffle(blockPeers)
+            peerUsed = blockPeers.pop(0)
+
+        if not comm_inst.shutdown and peerUsed.strip() != '':
+            logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
+        content = peeraction.peer_action(comm_inst, peerUsed, 'getdata/' + blockHash, max_resp_size=3000000) # block content from random peer (includes metadata)
+
+        if content != False and len(content) > 0:
+            try:
+                content = content.encode()
+            except AttributeError:
+                pass
+
+            realHash = onionrcrypto.hashers.sha3_hash(content)
+            try:
+                realHash = realHash.decode() # bytes on some versions for some reason
+            except AttributeError:
+                pass
+            if realHash == blockHash:
+                #content = content.decode() # decode here because sha3Hash needs bytes above
+                metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
+                metadata = metas[0]
+                try:
+                    metadata_validation_result = validatemetadata.validate_metadata(metadata, metas[2])
+                except onionrexceptions.DataExists:
+                    metadata_validation_result = False
+                if metadata_validation_result: # check if metadata is valid, and verify nonce
+                    if onionrcrypto.cryptoutils.verify_POW(content): # check if POW is enough/correct
+                        logger.info('Attempting to save block %s...' % blockHash[:12])
+                        try:
+                            onionrstorage.set_data(content)
+                        except onionrexceptions.DataExists:
+                            logger.warn('Data is already set for %s ' % (blockHash,))
+                        except onionrexceptions.DiskAllocationReached:
+                            logger.error('Reached disk allocation allowance, cannot save block %s.' % (blockHash,))
+                            removeFromQueue = False
+                        else:
+                            blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db
+                            blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
+                    else:
+                        logger.warn('POW failed for block %s.' % (blockHash,))
+                else:
+                    if blacklist.inBlacklist(realHash):
+                        logger.warn('Block %s is blacklisted.' % (realHash,))
+                    else:
+                        logger.warn('Metadata for block %s is invalid.' % (blockHash,))
+                        blacklist.addToDB(blockHash)
+            else:
+                # if block didn't meet expected hash
+                tempHash = onionrcrypto.hashers.sha3_hash(content) # lazy hack, TODO use var
+                try:
+                    tempHash = tempHash.decode()
+                except AttributeError:
+                    pass
+                # Punish peer for sharing invalid block (not always malicious, but is bad regardless)
+                onionrpeers.PeerProfiles(peerUsed).addScore(-50)
+                if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253':
+                    # Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error.
+                    logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
+                else:
+                    removeFromQueue = False # Don't remove from queue if 404
+            if removeFromQueue:
+                try:
+                    del comm_inst.blockQueue[blockHash] # remove from block queue both if success or false
+                    if count == LOG_SKIP_COUNT:
+                        logger.info('%s blocks remaining in queue' % [len(comm_inst.blockQueue)], terminal=True)
+                        count = 0
+                except KeyError:
+                    pass
+        comm_inst.currentDownloading.remove(blockHash)
+    comm_inst.decrementThreadCount('getBlocks')

src/communicatorutils/downloadblocks/shoulddownload.py

@@ -0,0 +1,37 @@
+'''
+    Onionr - Private P2P Communication
+
+    Check if a block should be downloaded (if we already have it or its blacklisted or not)
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from coredb import blockmetadb
+from onionrblocks import onionrblacklist
+
+def should_download(comm_inst, block_hash):
+    blacklist = onionrblacklist.OnionrBlackList()
+    ret_data = True
+    if block_hash in blockmetadb.get_block_list(): # Dont download block we have
+        ret_data = False
+    else:
+        if blacklist.inBlacklist(block_hash): # Dont download blacklisted block
+            ret_data = False
+    if ret_data is False:
+        # Remove block from communicator queue if it shouldnt be downloaded
+        try:
+            del comm_inst.blockQueue[block_hash]
+        except KeyError:
+            pass
+    return ret_data