efficiency improvements

master
Kevin Froman 2018-04-25 17:42:42 -05:00
parent 3aa539ce82
commit 66e55d5a50
No known key found for this signature in database
GPG Key ID: 0D414D0FE405B63B
4 changed files with 46 additions and 8 deletions

View File

@ -36,7 +36,7 @@ class OnionrCommunicate:
self._netController = netcontroller.NetController(0) # arg is the HS port but not needed rn in this file self._netController = netcontroller.NetController(0) # arg is the HS port but not needed rn in this file
self.newHashes = {} # use this to not keep hashes around too long if we cant get their data self.newHashes = {} # use this to not keep hashes around too long if we cant get their data
self.keepNewHash = 20 self.keepNewHash = 12
self.ignoredHashes = [] self.ignoredHashes = []
self.highFailureAmount = 7 self.highFailureAmount = 7
@ -182,6 +182,11 @@ class OnionrCommunicate:
peerList = self._core.listAdders() peerList = self._core.listAdders()
blocks = '' blocks = ''
for i in peerList: for i in peerList:
try:
if self.peerData[i]['failCount'] >= self.highFailureAmount:
continue
except KeyError:
pass
lastDB = self._core.getAddressInfo(i, 'DBHash') lastDB = self._core.getAddressInfo(i, 'DBHash')
if lastDB == None: if lastDB == None:
logger.debug('Fetching hash from ' + i + ' No previous known.') logger.debug('Fetching hash from ' + i + ' No previous known.')
@ -199,6 +204,7 @@ class OnionrCommunicate:
blocks += self.performGet('getBlockHashes', i) blocks += self.performGet('getBlockHashes', i)
except TypeError: except TypeError:
logger.warn('Failed to get data hash from ' + i) logger.warn('Failed to get data hash from ' + i)
self.peerData[peer]['failCount'] -= 1
if self._utils.validateHash(currentDB): if self._utils.validateHash(currentDB):
self._core.setAddressInfo(i, "DBHash", currentDB) self._core.setAddressInfo(i, "DBHash", currentDB)
if len(blocks.strip()) != 0: if len(blocks.strip()) != 0:
@ -252,17 +258,23 @@ class OnionrCommunicate:
del self.newHashes[i] del self.newHashes[i]
return return
def downloadBlock(self, hash): def downloadBlock(self, hash, peerTries=3):
''' '''
Download a block from random order of peers Download a block from random order of peers
''' '''
retVal = False retVal = False
peerList = self._core.listAdders() peerList = self._core.listAdders()
blocks = '' blocks = ''
peerTryCount = 0
for i in peerList: for i in peerList:
if self.peerData[i]['failCount'] >= self.highFailureAmount:
continue
if peerTryCount >= peerTries:
break
hasher = hashlib.sha3_256() hasher = hashlib.sha3_256()
data = self.performGet('getData', i, hash, skipHighFailureAddress=True) data = self.performGet('getData', i, hash, skipHighFailureAddress=True)
if data == False or len(data) > 10000000 or data == '': if data == False or len(data) > 10000000 or data == '':
peerTryCount += 1
continue continue
try: try:
data = base64.b64decode(data) data = base64.b64decode(data)
@ -282,6 +294,7 @@ class OnionrCommunicate:
logger.debug('Block text:\n' + data.decode()) logger.debug('Block text:\n' + data.decode())
else: else:
logger.warn("Failed to validate " + hash + " " + " hash calculated was " + digest) logger.warn("Failed to validate " + hash + " " + " hash calculated was " + digest)
peerTryCount += 1
return retVal return retVal

View File

@ -552,9 +552,9 @@ class Core:
c = conn.cursor() c = conn.cursor()
retData = '' retData = ''
if unsaved: if unsaved:
execute = 'SELECT hash FROM hashes WHERE dataSaved != 1;' execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
else: else:
execute = 'SELECT hash FROM hashes;' execute = 'SELECT hash FROM hashes ORDER BY RANDOM();'
for row in c.execute(execute): for row in c.execute(execute):
for i in row: for i in row:
retData += i + "\n" retData += i + "\n"
@ -594,10 +594,7 @@ class Core:
Inserts a block into the network Inserts a block into the network
''' '''
retData = '' retData = ''
metadata = header metadata = '-' + header + '-'
if sign:
metadata += '-' + self._crypto.pubKeyHashID() + '-'
metadata += self._crypto.edSign(data, encodeResult=True) + '-'
if len(data) == 0: if len(data) == 0:
logger.error('Will not insert empty block') logger.error('Will not insert empty block')
else: else:

26
onionr/onionrblocks.py Normal file
View File

@ -0,0 +1,26 @@
'''
Onionr - P2P Microblogging Platform & Social network.
This class contains the OnionrBlocks class which is a class for working with Onionr blocks
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import json
class OnionrBlocks:
def __init__(self, coreInstance):
return
def metadataGenerate(self):
return

View File

@ -333,6 +333,8 @@ class OnionrUtils:
pass pass
else: else:
logger.info('Recieved message: ' + message.decode()) logger.info('Recieved message: ' + message.decode())
except FileNotFoundError:
pass
except Exception as error: except Exception as error:
logger.error('Failed to open block ' + str(i) + '.', error=error) logger.error('Failed to open block ' + str(i) + '.', error=error)
return return