diff --git a/onionr/communicator.py b/onionr/communicator.py index 521f93b6..2b342102 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -301,7 +301,7 @@ class OnionrCommunicatorDaemon: score = str(self.getPeerProfileInstance(i).score) logger.info(i + ', score: ' + score, terminal=True) - def peerAction(self, peer, action, data='', returnHeaders=False): + def peerAction(self, peer, action, data='', returnHeaders=False, max_resp_size=5242880): '''Perform a get request to a peer''' if len(peer) == 0: return False diff --git a/onionr/communicatorutils/downloadblocks.py b/onionr/communicatorutils/downloadblocks/__init__.py similarity index 90% rename from onionr/communicatorutils/downloadblocks.py rename to onionr/communicatorutils/downloadblocks/__init__.py index 6d1c8daf..f8d7fdd5 100755 --- a/onionr/communicatorutils/downloadblocks.py +++ b/onionr/communicatorutils/downloadblocks/__init__.py @@ -20,20 +20,7 @@ import communicator, onionrexceptions import logger, onionrpeers from onionrutils import blockmetadata, stringvalidators, validatemetadata - -def _should_download(comm_inst, block_hash): - ret_data = True - if block_hash in comm_inst._core.getBlockList(): - ret_data = False - else: - if comm_inst._core._blacklist.inBlacklist(block_hash): - ret_data = False - if ret_data is False: - try: - del comm_inst.blockQueue[block_hash] - except KeyError: - pass - return ret_data +from . import shoulddownload def download_blocks_from_communicator(comm_inst): assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon) @@ -47,13 +34,13 @@ def download_blocks_from_communicator(comm_inst): blockPeers = [] removeFromQueue = True - if _should_download(comm_inst, blockHash): + if shoulddownload.should_download(comm_inst, blockHash): continue if comm_inst.shutdown or not comm_inst.isOnline or comm_inst._core.storage_counter.isFull(): # Exit loop if shutting down or offline, or disk allocation reached break - # Do not download blocks being downloaded or that are already saved (edge cases) + # Do not download blocks being downloaded if blockHash in comm_inst.currentDownloading: #logger.debug('Already downloading block %s...' % blockHash) continue @@ -67,7 +54,7 @@ def download_blocks_from_communicator(comm_inst): if not comm_inst.shutdown and peerUsed.strip() != '': logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed)) - content = comm_inst.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata) + content = comm_inst.peerAction(peerUsed, 'getdata/' + blockHash, max_resp_size=3000000) # block content from random peer (includes metadata) if content != False and len(content) > 0: try: content = content.encode() diff --git a/onionr/communicatorutils/downloadblocks/shoulddownload.py b/onionr/communicatorutils/downloadblocks/shoulddownload.py new file mode 100644 index 00000000..d126f548 --- /dev/null +++ b/onionr/communicatorutils/downloadblocks/shoulddownload.py @@ -0,0 +1,33 @@ +''' + Onionr - Private P2P Communication + + Check if a block should be downloaded (if we already have it or its blacklisted or not) +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +def should_download(comm_inst, block_hash): + ret_data = True + if block_hash in comm_inst._core.getBlockList(): # Dont download block we have + ret_data = False + else: + if comm_inst._core._blacklist.inBlacklist(block_hash): # Dont download blacklisted block + ret_data = False + if ret_data is False: + # Remove block from communicator queue if it shouldnt be downloaded + try: + del comm_inst.blockQueue[block_hash] + except KeyError: + pass + return ret_data \ No newline at end of file diff --git a/onionr/onionrutils/basicrequests.py b/onionr/onionrutils/basicrequests.py index 1595b433..8bb85279 100644 --- a/onionr/onionrutils/basicrequests.py +++ b/onionr/onionrutils/basicrequests.py @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import requests +import requests, streamedrequests import logger, onionrexceptions def do_post_request(core_inst, url, data={}, port=0, proxyType='tor'): ''' @@ -43,7 +43,7 @@ def do_post_request(core_inst, url, data={}, port=0, proxyType='tor'): retData = False return retData -def do_get_request(core_inst, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False): +def do_get_request(core_inst, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880): ''' Do a get request through a local tor or i2p instance ''' @@ -61,16 +61,16 @@ def do_get_request(core_inst, url, port=0, proxyType='tor', ignoreAPI=False, ret response_headers = dict() try: proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)} - r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30), ) + r = streamedrequests.get(url, request_headers=headers, allow_redirects=False, proxy=proxies, connect_timeout=15, max_size=max_size) # Check server is using same API version as us if not ignoreAPI: try: - response_headers = r.headers - if r.headers['X-API'] != str(API_VERSION): + response_headers = r[0].headers + if r[0].headers['X-API'] != str(API_VERSION): raise onionrexceptions.InvalidAPIVersion except KeyError: raise onionrexceptions.InvalidAPIVersion - retData = r.text + retData = r[1] except KeyboardInterrupt: raise KeyboardInterrupt except ValueError as e: diff --git a/requirements.in b/requirements.in index 4b9b6836..3b90933e 100644 --- a/requirements.in +++ b/requirements.in @@ -7,5 +7,5 @@ PySocks==1.6.8 stem==1.7.1 deadsimplekv==0.1.1 unpaddedbase32==0.1.0 -streamedrequests==0.0.0 +streamedrequests==1.0.0 jinja2==2.10.1 diff --git a/requirements.txt b/requirements.txt index 747ee2d5..b5ca0af4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -171,8 +171,8 @@ six==1.12.0 \ # via pynacl stem==1.7.1 \ --hash=sha256:c9eaf3116cb60c15995cbd3dec3a5cbc50e9bb6e062c4d6d42201e566f498ca2 -streamedrequests==0.0.0 \ - --hash=sha256:94c729078099c0f2691510be0bb90f421277a729078b8e150fe2740c78f8e53e +streamedrequests==1.0.0 \ + --hash=sha256:1d9d07394804a6e1fd66bde74a804e71cab98e6920053865574a459f1cf7d3b7 unpaddedbase32==0.1.0 \ --hash=sha256:5e4143fcaf77c9c6b4f60d18301c7570f0dac561dcf9b9aed8b5ba6ead7f218c urllib3==1.24.2 \