onionr now removes and ignores blocks it cant get after a long time, added removeblock function

master
Kevin Froman 2018-04-23 01:03:10 -05:00
parent db2babac29
commit 0db21e2c32
No known key found for this signature in database
GPG Key ID: 0D414D0FE405B63B
3 changed files with 43 additions and 6 deletions

View File

@ -35,6 +35,10 @@ class OnionrCommunicate:
self._crypto = onionrcrypto.OnionrCrypto(self._core) self._crypto = onionrcrypto.OnionrCrypto(self._core)
self._netController = netcontroller.NetController(0) # arg is the HS port but not needed rn in this file self._netController = netcontroller.NetController(0) # arg is the HS port but not needed rn in this file
self.newHashes = {} # use this to not keep hashes around too long if we cant get their data
self.keepNewHash = 100
self.ignoredHashes = []
self.highFailureAmount = 7 self.highFailureAmount = 7
''' '''
logger.info('Starting Bitcoin Node... with Tor socks port:' + str(sys.argv[2]), timestamp=True) logger.info('Starting Bitcoin Node... with Tor socks port:' + str(sys.argv[2]), timestamp=True)
@ -210,6 +214,7 @@ class OnionrCommunicate:
logger.warn('Hash ' + i + ' is not valid') logger.warn('Hash ' + i + ' is not valid')
continue continue
else: else:
self.newHashes[i] = 0
logger.debug('Adding ' + i + ' to hash database...') logger.debug('Adding ' + i + ' to hash database...')
self._core.addToBlockDB(i) self._core.addToBlockDB(i)
@ -222,24 +227,39 @@ class OnionrCommunicate:
This is meant to be called from the communicator daemon on its timer. This is meant to be called from the communicator daemon on its timer.
''' '''
for i in self._core.getBlockList(True).split("\n"): for i in self._core.getBlockList(unsaved=True).split("\n"):
if i != "": if i != "":
if i in self.ignoredHashes:
continue
try:
self.newHashes[i]
except KeyError:
self.newHashes[i] = 0
# check if a new hash has been around too long, delete it from database and add it to ignore list
if self.newHashes[i] >= self.keepNewHash:
logger.warn('Ignoring block ' + i + ' because it took to long to get valid data.')
del self.newHashes[i]
self._core.removeBlock(i)
self.ignoredHashes.append(i)
continue
self.newHashes[i] += 1
logger.warn('UNSAVED BLOCK: ' + i) logger.warn('UNSAVED BLOCK: ' + i)
data = self.downloadBlock(i) data = self.downloadBlock(i)
if data:
del self.newHashes[i]
return return
def downloadBlock(self, hash): def downloadBlock(self, hash):
''' '''
Download a block from random order of peers Download a block from random order of peers
''' '''
retVal = False
peerList = self._core.listAdders() peerList = self._core.listAdders()
blocks = '' blocks = ''
for i in peerList: for i in peerList:
hasher = hashlib.sha3_256() hasher = hashlib.sha3_256()
data = self.performGet('getData', i, hash) data = self.performGet('getData', i, hash)
if data == False or len(data) > 10000000: if data == False or len(data) > 10000000 or data == '':
continue continue
try: try:
data = base64.b64decode(data) data = base64.b64decode(data)
@ -252,6 +272,7 @@ class OnionrCommunicate:
if digest == hash.strip(): if digest == hash.strip():
self._core.setData(data) self._core.setData(data)
logger.info('Successfully obtained data for ' + hash, timestamp=True) logger.info('Successfully obtained data for ' + hash, timestamp=True)
retVal = True
if data.startswith(b'-txt-'): if data.startswith(b'-txt-'):
self._core.setBlockType(hash, 'txt') self._core.setBlockType(hash, 'txt')
if len(data) < 120: if len(data) < 120:
@ -259,7 +280,7 @@ class OnionrCommunicate:
else: else:
logger.warn("Failed to validate " + hash + " " + " hash calculated was " + digest) logger.warn("Failed to validate " + hash + " " + " hash calculated was " + digest)
return return retVal
def urlencode(self, data): def urlencode(self, data):
''' '''

View File

@ -154,6 +154,22 @@ class Core:
return True return True
else: else:
return False return False
def removeBlock(self, block):
'''
remove a block from this node
'''
if self._utils.validateHash(block):
conn = sqlite3.connect(self.blockDB)
c = conn.cursor()
t = (block,)
c.execute('Delete from hashes where hash=?;', t)
conn.commit()
conn.close()
try:
os.remove('data/blocks/' + block + '.dat')
except FileNotFoundError:
pass
def createAddressDB(self): def createAddressDB(self):
''' '''

View File

@ -335,4 +335,4 @@ class OnionrUtils:
logger.info('Recieved message: ' + message.decode()) logger.info('Recieved message: ' + message.decode())
except Exception as error: except Exception as error:
logger.error('Failed to open block ' + str(i) + '.', error=error) logger.error('Failed to open block ' + str(i) + '.', error=error)
return return