based block size protection added

master
Kevin Froman 2019-07-09 01:25:20 -05:00
parent fe142528ca
commit b822a74c53
6 changed files with 47 additions and 27 deletions

View File

@ -301,7 +301,7 @@ class OnionrCommunicatorDaemon:
score = str(self.getPeerProfileInstance(i).score) score = str(self.getPeerProfileInstance(i).score)
logger.info(i + ', score: ' + score, terminal=True) logger.info(i + ', score: ' + score, terminal=True)
def peerAction(self, peer, action, data='', returnHeaders=False): def peerAction(self, peer, action, data='', returnHeaders=False, max_resp_size=5242880):
'''Perform a get request to a peer''' '''Perform a get request to a peer'''
if len(peer) == 0: if len(peer) == 0:
return False return False

View File

@ -20,20 +20,7 @@
import communicator, onionrexceptions import communicator, onionrexceptions
import logger, onionrpeers import logger, onionrpeers
from onionrutils import blockmetadata, stringvalidators, validatemetadata from onionrutils import blockmetadata, stringvalidators, validatemetadata
from . import shoulddownload
def _should_download(comm_inst, block_hash):
ret_data = True
if block_hash in comm_inst._core.getBlockList():
ret_data = False
else:
if comm_inst._core._blacklist.inBlacklist(block_hash):
ret_data = False
if ret_data is False:
try:
del comm_inst.blockQueue[block_hash]
except KeyError:
pass
return ret_data
def download_blocks_from_communicator(comm_inst): def download_blocks_from_communicator(comm_inst):
assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon) assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon)
@ -47,13 +34,13 @@ def download_blocks_from_communicator(comm_inst):
blockPeers = [] blockPeers = []
removeFromQueue = True removeFromQueue = True
if _should_download(comm_inst, blockHash): if shoulddownload.should_download(comm_inst, blockHash):
continue continue
if comm_inst.shutdown or not comm_inst.isOnline or comm_inst._core.storage_counter.isFull(): if comm_inst.shutdown or not comm_inst.isOnline or comm_inst._core.storage_counter.isFull():
# Exit loop if shutting down or offline, or disk allocation reached # Exit loop if shutting down or offline, or disk allocation reached
break break
# Do not download blocks being downloaded or that are already saved (edge cases) # Do not download blocks being downloaded
if blockHash in comm_inst.currentDownloading: if blockHash in comm_inst.currentDownloading:
#logger.debug('Already downloading block %s...' % blockHash) #logger.debug('Already downloading block %s...' % blockHash)
continue continue
@ -67,7 +54,7 @@ def download_blocks_from_communicator(comm_inst):
if not comm_inst.shutdown and peerUsed.strip() != '': if not comm_inst.shutdown and peerUsed.strip() != '':
logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed)) logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
content = comm_inst.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata) content = comm_inst.peerAction(peerUsed, 'getdata/' + blockHash, max_resp_size=3000000) # block content from random peer (includes metadata)
if content != False and len(content) > 0: if content != False and len(content) > 0:
try: try:
content = content.encode() content = content.encode()

View File

@ -0,0 +1,33 @@
'''
Onionr - Private P2P Communication
Check if a block should be downloaded (if we already have it or its blacklisted or not)
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
def should_download(comm_inst, block_hash):
ret_data = True
if block_hash in comm_inst._core.getBlockList(): # Dont download block we have
ret_data = False
else:
if comm_inst._core._blacklist.inBlacklist(block_hash): # Dont download blacklisted block
ret_data = False
if ret_data is False:
# Remove block from communicator queue if it shouldnt be downloaded
try:
del comm_inst.blockQueue[block_hash]
except KeyError:
pass
return ret_data

View File

@ -17,7 +17,7 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import requests import requests, streamedrequests
import logger, onionrexceptions import logger, onionrexceptions
def do_post_request(core_inst, url, data={}, port=0, proxyType='tor'): def do_post_request(core_inst, url, data={}, port=0, proxyType='tor'):
''' '''
@ -43,7 +43,7 @@ def do_post_request(core_inst, url, data={}, port=0, proxyType='tor'):
retData = False retData = False
return retData return retData
def do_get_request(core_inst, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False): def do_get_request(core_inst, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880):
''' '''
Do a get request through a local tor or i2p instance Do a get request through a local tor or i2p instance
''' '''
@ -61,16 +61,16 @@ def do_get_request(core_inst, url, port=0, proxyType='tor', ignoreAPI=False, ret
response_headers = dict() response_headers = dict()
try: try:
proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)} proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30), ) r = streamedrequests.get(url, request_headers=headers, allow_redirects=False, proxy=proxies, connect_timeout=15, max_size=max_size)
# Check server is using same API version as us # Check server is using same API version as us
if not ignoreAPI: if not ignoreAPI:
try: try:
response_headers = r.headers response_headers = r[0].headers
if r.headers['X-API'] != str(API_VERSION): if r[0].headers['X-API'] != str(API_VERSION):
raise onionrexceptions.InvalidAPIVersion raise onionrexceptions.InvalidAPIVersion
except KeyError: except KeyError:
raise onionrexceptions.InvalidAPIVersion raise onionrexceptions.InvalidAPIVersion
retData = r.text retData = r[1]
except KeyboardInterrupt: except KeyboardInterrupt:
raise KeyboardInterrupt raise KeyboardInterrupt
except ValueError as e: except ValueError as e:

View File

@ -7,5 +7,5 @@ PySocks==1.6.8
stem==1.7.1 stem==1.7.1
deadsimplekv==0.1.1 deadsimplekv==0.1.1
unpaddedbase32==0.1.0 unpaddedbase32==0.1.0
streamedrequests==0.0.0 streamedrequests==1.0.0
jinja2==2.10.1 jinja2==2.10.1

View File

@ -171,8 +171,8 @@ six==1.12.0 \
# via pynacl # via pynacl
stem==1.7.1 \ stem==1.7.1 \
--hash=sha256:c9eaf3116cb60c15995cbd3dec3a5cbc50e9bb6e062c4d6d42201e566f498ca2 --hash=sha256:c9eaf3116cb60c15995cbd3dec3a5cbc50e9bb6e062c4d6d42201e566f498ca2
streamedrequests==0.0.0 \ streamedrequests==1.0.0 \
--hash=sha256:94c729078099c0f2691510be0bb90f421277a729078b8e150fe2740c78f8e53e --hash=sha256:1d9d07394804a6e1fd66bde74a804e71cab98e6920053865574a459f1cf7d3b7
unpaddedbase32==0.1.0 \ unpaddedbase32==0.1.0 \
--hash=sha256:5e4143fcaf77c9c6b4f60d18301c7570f0dac561dcf9b9aed8b5ba6ead7f218c --hash=sha256:5e4143fcaf77c9c6b4f60d18301c7570f0dac561dcf9b9aed8b5ba6ead7f218c
urllib3==1.24.2 \ urllib3==1.24.2 \