From 16282d79d394bc379c4c30d72efec84a6d5fb6a0 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Fri, 18 May 2018 01:22:16 -0500 Subject: [PATCH] refactored timers and added lastconnect to address database --- onionr/communicator.py | 76 ++++++++++++++++---------- onionr/core.py | 10 ++-- onionr/netcontroller.py | 18 +++--- onionr/onionrutils.py | 6 +- onionr/static-data/default_config.json | 4 ++ 5 files changed, 73 insertions(+), 41 deletions(-) diff --git a/onionr/communicator.py b/onionr/communicator.py index 6f531e47..777660ad 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -49,16 +49,18 @@ class OnionrCommunicate: self.blocksProcessing = [] # list of blocks currently processing, to avoid trying a block twice at once in 2 seperate threads self.peerStatus = {} # network actions (active requests) for peers used mainly to prevent conflicting actions in threads - blockProcessTimer = 19 - blockProcessAmount = 20 - highFailureTimer = 0 - highFailureRate = 10 - heartBeatTimer = 0 - heartBeatRate = 10 - pexTimer = 120 # How often we should check for new peers - pexCount = 0 + self.communicatorTimers = {} # communicator timers, name: rate (in seconds) + self.communicatorTimerCounts = {} + self.communicatorTimerFuncs = {} + + self.registerTimer('blockProcess', 20) + self.registerTimer('highFailure', 10) + self.registerTimer('heartBeat', 10) + self.registerTimer('pex', 120) logger.debug('Communicator debugging enabled.') - torID = open('data/hs/hostname').read() + + with open('data/hs/hostname', 'r') as torID: + todID = torID.read() apiRunningCheckRate = 10 apiRunningCheckCount = 0 @@ -74,24 +76,23 @@ class OnionrCommunicate: while True: command = self._core.daemonQueue() # Process blocks based on a timer - blockProcessTimer += 1 - heartBeatTimer += 1 - pexCount += 1 - if highFailureTimer == highFailureRate: - highFailureTimer = 0 + self.timerTick() + # TODO: migrate below if statements to be own functions which are called in the above timerTick() function + if self.communicatorTimers['highFailure'] == self.communicatorTimerCounts['highFailure']: + self.communicatorTimerCounts['highFailure'] = 0 for i in self.peerData: if self.peerData[i]['failCount'] >= self.highFailureAmount: self.peerData[i]['failCount'] -= 1 - if pexTimer == pexCount: + if self.communicatorTimers['pex'] == self.communicatorTimerCounts['pex']: pT1 = threading.Thread(target=self.getNewPeers, name="pT1") pT1.start() pT2 = threading.Thread(target=self.getNewPeers, name="pT2") pT2.start() - pexCount = 0 # TODO: do not reset timer if low peer count - if heartBeatRate == heartBeatTimer: + self.communicatorTimerCounts['pex'] = 0# TODO: do not reset timer if low peer count + if self.communicatorTimers['heartBeat'] == self.communicatorTimerCounts['heartBeat']: logger.debug('Communicator heartbeat') - heartBeatTimer = 0 - if blockProcessTimer == blockProcessAmount: + self.communicatorTimerCounts['heartBeat'] = 0 + if self.communicatorTimers['blockProcess'] == self.communicatorTimerCounts['blockProcess']: lT1 = threading.Thread(target=self.lookupBlocks, name="lt1", args=(True,)) lT2 = threading.Thread(target=self.lookupBlocks, name="lt2", args=(True,)) lT3 = threading.Thread(target=self.lookupBlocks, name="lt3", args=(True,)) @@ -109,7 +110,7 @@ class OnionrCommunicate: pbT2.start() pbT3.start() pbT4.start() - blockProcessTimer = 0 + self.communicatorTimerCounts['blockProcess'] = 0 else: logger.debug(threading.active_count()) logger.debug('Too many threads.') @@ -185,6 +186,28 @@ class OnionrCommunicate: connection_handlers = {} id_peer_cache = {} + def registerTimer(self, timerName, rate, timerFunc=None): + '''Register a communicator timer''' + self.communicatorTimers[timerName] = rate + self.communicatorTimerCounts[timerName] = 0 + self.communicatorTimerFuncs[timerName] = timerFunc + + def timerTick(self): + '''Increments timers "ticks" and calls funcs if applicable''' + tName = '' + for i in self.communicatorTimers.items(): + tName = i[0] + self.communicatorTimerCounts[tName] += 1 + + if self.communicatorTimerCounts[tName] == self.communicatorTimers[tName]: + try: + self.communicatorTimerFuncs[tName]() + except TypeError: + pass + else: + self.communicatorTimerCounts[tName] = 0 + + def get_connection_handlers(self, name = None): ''' Returns a list of callback handlers by name, or, if name is None, it returns all handlers. @@ -637,12 +660,6 @@ class OnionrCommunicate: logger.info('Successfully obtained data for %s' % str(hash), timestamp=True) retVal = True break - ''' - if data.startswith(b'-txt-'): - self._core.setBlockType(hash, 'txt') - if len(data) < 120: - logger.debug('Block text:\n' + data.decode()) - ''' else: logger.warn("Failed to validate %s -- hash calculated was %s" % (hash, digest)) peerTryCount += 1 @@ -672,7 +689,7 @@ class OnionrCommunicate: # Store peer in peerData dictionary (non permanent) if not peer in self.peerData: - self.peerData[peer] = {'connectCount': 0, 'failCount': 0, 'lastConnectTime': math.floor(time.time())} + self.peerData[peer] = {'connectCount': 0, 'failCount': 0, 'lastConnectTime': self._utils.getEpoch()} socksPort = sys.argv[2] '''We use socks5h to use tor as DNS''' proxies = {'http': 'socks5://127.0.0.1:' + str(socksPort), 'https': 'socks5://127.0.0.1:' + str(socksPort)} @@ -698,12 +715,13 @@ class OnionrCommunicate: else: self.peerData[peer]['connectCount'] += 1 self.peerData[peer]['failCount'] -= 1 - self.peerData[peer]['lastConnectTime'] = math.floor(time.time()) + self.peerData[peer]['lastConnectTime'] = self._utils.getEpoch() + self._core.setAddressInfo(peer, 'lastConnect', self._utils.getEpoch()) return retData def peerStatusTaken(self, peer, status): ''' - Returns if a peer is currently performing a specified action + Returns if we are currently performing a specific action with a peer. ''' try: if self.peerStatus[peer] == status: diff --git a/onionr/core.py b/onionr/core.py index d7b00ea5..57c1d53f 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -188,7 +188,8 @@ class Core: speed int, success int, DBHash text, - failure int + failure int, + lastConnect int ); ''') conn.commit() @@ -263,7 +264,7 @@ class Core: return conn = sqlite3.connect(self.blockDB) c = conn.cursor() - currentTime = math.floor(time.time()) + currentTime = self._utils.getEpoch() if selfInsert or dataSaved: selfInsert = 1 else: @@ -388,7 +389,7 @@ class Core: Add a command to the daemon queue, used by the communication daemon (communicator.py) ''' # Intended to be used by the web server - date = math.floor(time.time()) + date = self._utils.getEpoch() conn = sqlite3.connect(self.queueDB) c = conn.cursor() t = (command, data, date) @@ -523,11 +524,12 @@ class Core: success int, 4 DBHash text, 5 failure int 6 + lastConnect 7 ''' conn = sqlite3.connect(self.addressDB) c = conn.cursor() command = (address,) - infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'DBHash': 5, 'failure': 6} + infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'DBHash': 5, 'failure': 6, 'lastConnect': 7} info = infoNumbers[info] iterCount = 0 retVal = '' diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index 77149477..4c25a7fc 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -89,13 +89,17 @@ DataDirectory data/tordata/ torVersion.kill() # wait for tor to get to 100% bootstrap - for line in iter(tor.stdout.readline, b''): - if 'Bootstrapped 100%: Done' in line.decode(): - break - elif 'Opening Socks listener' in line.decode(): - logger.debug(line.decode().replace('\n', '')) - else: - logger.fatal('Failed to start Tor. Try killing any other Tor processes owned by this user.') + try: + for line in iter(tor.stdout.readline, b''): + if 'Bootstrapped 100%: Done' in line.decode(): + break + elif 'Opening Socks listener' in line.decode(): + logger.debug(line.decode().replace('\n', '')) + else: + logger.fatal('Failed to start Tor. Try killing any other Tor processes owned by this user.') + return False + except KeyboardInterrupt: + logger.fatal("Got keyboard interrupt") return False logger.info('Finished starting Tor', timestamp=True) diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 4badd1d1..317ff8c5 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -472,6 +472,10 @@ class OnionrUtils: sys.stdout.write("\r┣{0}┫ {1}%".format(arrow + spaces, int(round(percent * 100)))) sys.stdout.flush() + + def getEpoch(): + '''returns epoch''' + return math.floor(time.time()) def size(path='.'): ''' @@ -497,4 +501,4 @@ def humanSize(num, suffix='B'): if abs(num) < 1024.0: return "%.1f %s%s" % (num, unit, suffix) num /= 1024.0 - return "%.1f %s%s" % (num, 'Yi', suffix) + return "%.1f %s%s" % (num, 'Yi', suffix) \ No newline at end of file diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 020107ce..ccafd7a4 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -12,5 +12,9 @@ "output": true, "color": true } + }, + "allocations":{ + "disk": 1000000000, + "netTotal": 1000000000 } }