From f28d469e56676e3b0f3185c12c1c9bd382e532ef Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 29 Jul 2020 03:57:06 -0500 Subject: [PATCH] Moved all communicator ext vars to KV --- src/communicator/__init__.py | 29 +++++++------------ src/communicator/onlinepeers/onlinepeers.py | 9 +++--- .../onlinepeers/removeonlinepeer.py | 2 +- src/communicatorutils/connectnewpeers.py | 6 ++-- src/communicatorutils/cooldownpeer.py | 2 +- .../uploadblocks/sessionmanager.py | 5 ++-- src/onionrstatistics/serializeddata.py | 16 +++++++--- 7 files changed, 34 insertions(+), 35 deletions(-) diff --git a/src/communicator/__init__.py b/src/communicator/__init__.py index dcb5525f..b887e6fa 100755 --- a/src/communicator/__init__.py +++ b/src/communicator/__init__.py @@ -10,7 +10,8 @@ import config import logger import onionrpeers import onionrplugins as plugins -from . import onlinepeers, uploadqueue +from . import onlinepeers +from . import uploadqueue from communicatorutils import servicecreator from communicatorutils import onionrcommunicatortimers from communicatorutils import downloadblocks @@ -65,6 +66,8 @@ class OnionrCommunicatorDaemon: self.kv.put('shutdown', False) self.kv.put('onlinePeers', []) self.kv.put('offlinePeers', []) + self.kv.put('peerProfiles', []) + self.kv.put('connectTimes', {}) self.kv.put('currentDownloading', []) self.kv.put('announceCache', {}) self.kv.put('newPeers', []) @@ -72,6 +75,8 @@ class OnionrCommunicatorDaemon: self.kv.put('blocksToUpload', []) self.kv.put('cooldownPeer', {}) self.kv.put('generating_blocks', []) + self.kv.put('lastNodeSeen', None) + self.kv.put('startTime', epoch.get_epoch()) if config.get('general.offline_mode', False): self.isOnline = False @@ -89,23 +94,12 @@ class OnionrCommunicatorDaemon: # loop time.sleep delay in seconds self.delay = 1 - # lists of connected peers and peers we know we can't reach currently - self.connectTimes = {} - # list of peer's profiles (onionrpeers.PeerProfile instances) - self.peerProfiles = [] - # amount of threads running by name, used to prevent too many self.threadCounts = {} - # timestamp when the last online node was seen - self.lastNodeSeen = None - # Loads in and starts the enabled plugins plugins.reload() - # time app started running for info/statistics purposes - self.startTime = epoch.get_epoch() - # extends our upload list and saves our list when Onionr exits uploadqueue.UploadQueue(self) @@ -296,7 +290,7 @@ class OnionrCommunicatorDaemon: def getPeerProfileInstance(self, peer): """Gets a peer profile instance from the list of profiles""" - for i in self.peerProfiles: + for i in self.kv.get('peerProfiles'): # if the peer's profile is already loaded, return that if i.address == peer: retData = i @@ -305,19 +299,16 @@ class OnionrCommunicatorDaemon: # if the peer's profile is not loaded, return a new one. # connectNewPeer also adds it to the list on connect retData = onionrpeers.PeerProfiles(peer) - self.peerProfiles.append(retData) + self.kv.get('peerProfiles').append(retData) return retData - def getUptime(self): - return epoch.get_epoch() - self.startTime - def heartbeat(self): """Show a heartbeat debug message.""" logger.debug('Heartbeat. Node running for %s.' % - humanreadabletime.human_readable_time(self.getUptime())) + humanreadabletime.human_readable_time( + self.kv.get('startTime'))) self.decrementThreadCount('heartbeat') def startCommunicator(shared_state): OnionrCommunicatorDaemon(shared_state) - diff --git a/src/communicator/onlinepeers/onlinepeers.py b/src/communicator/onlinepeers/onlinepeers.py index c9cb6026..c616e8b0 100644 --- a/src/communicator/onlinepeers/onlinepeers.py +++ b/src/communicator/onlinepeers/onlinepeers.py @@ -5,7 +5,7 @@ get online peers in a communicator instance import time from typing import TYPE_CHECKING -from etc import humanreadabletime +from etc.humanreadabletime import human_readable_time import logger if TYPE_CHECKING: from deadsimplekv import DeadSimpleKV @@ -41,9 +41,8 @@ def get_online_peers(comm_inst: 'OnionrCommunicatorDaemon'): needed = max_peers - len(kv.get('onlinePeers')) last_seen = 'never' - if not isinstance(comm_inst.lastNodeSeen, type(None)): - last_seen = humanreadabletime.human_readable_time( - comm_inst.lastNodeSeen) + if not isinstance(kv.get('lastNodeSeen'), type(None)): + last_seen = human_readable_time(kv.get('lastNodeSeen')) for _ in range(needed): if len(kv.get('onlinePeers')) == 0: @@ -62,5 +61,5 @@ def get_online_peers(comm_inst: 'OnionrCommunicatorDaemon'): except RecursionError: pass else: - comm_inst.lastNodeSeen = time.time() + kv.put('lastNodeSeen', time.time()) comm_inst.decrementThreadCount('get_online_peers') diff --git a/src/communicator/onlinepeers/removeonlinepeer.py b/src/communicator/onlinepeers/removeonlinepeer.py index 60db6ca3..fcd42328 100644 --- a/src/communicator/onlinepeers/removeonlinepeer.py +++ b/src/communicator/onlinepeers/removeonlinepeer.py @@ -26,7 +26,7 @@ def remove_online_peer(comm_inst, peer): """Remove an online peer.""" kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV") try: - del comm_inst.connectTimes[peer] + del kv.get('connectTimes')[peer] except KeyError: pass try: diff --git a/src/communicatorutils/connectnewpeers.py b/src/communicatorutils/connectnewpeers.py index 7beb9e9c..a660e5ed 100755 --- a/src/communicatorutils/connectnewpeers.py +++ b/src/communicatorutils/connectnewpeers.py @@ -90,15 +90,15 @@ def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False): if address not in kv.get('onlinePeers'): logger.info('Connected to ' + address, terminal=True) kv.get('onlinePeers').append(address) - comm_inst.connectTimes[address] = epoch.get_epoch() + kv.get('connectTimes')[address] = epoch.get_epoch() retData = address # add peer to profile list if they're not in it - for profile in comm_inst.peerProfiles: + for profile in kv.get('peerProfiles'): if profile.address == address: break else: - comm_inst.peerProfiles.append( + kv.get('peerProfiles').append( onionrpeers.PeerProfiles(address)) break else: diff --git a/src/communicatorutils/cooldownpeer.py b/src/communicatorutils/cooldownpeer.py index 969ba7a3..639d92aa 100755 --- a/src/communicatorutils/cooldownpeer.py +++ b/src/communicatorutils/cooldownpeer.py @@ -33,7 +33,7 @@ def cooldown_peer(comm_inst): minTime = 300 cooldown_time = 600 to_cool = '' - tempConnectTimes = dict(comm_inst.connectTimes) + tempConnectTimes = dict(kv.get('connectTimes')) # Remove peers from cooldown that have been there long enough tempCooldown = dict(kv.get('cooldownPeer')) diff --git a/src/communicatorutils/uploadblocks/sessionmanager.py b/src/communicatorutils/uploadblocks/sessionmanager.py index c02291a3..486823a3 100644 --- a/src/communicatorutils/uploadblocks/sessionmanager.py +++ b/src/communicatorutils/uploadblocks/sessionmanager.py @@ -88,7 +88,7 @@ class BlockUploadSessionManager: kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string( "DeadSimpleKV") sessions_to_delete = [] - if comm_inst.getUptime() < 120: + if kv.get('startTime') < 120: return onlinePeerCount = len(kv.get('onlinePeers')) @@ -105,7 +105,8 @@ class BlockUploadSessionManager: # Clean sessions if they have uploaded to enough online peers if sess.total_success_count <= 0: continue - if (sess.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT: + if (sess.total_success_count / onlinePeerCount) >= \ + onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT: sessions_to_delete.append(sess) for sess in sessions_to_delete: try: diff --git a/src/onionrstatistics/serializeddata.py b/src/onionrstatistics/serializeddata.py index 5a13d4ed..005f7ed0 100755 --- a/src/onionrstatistics/serializeddata.py +++ b/src/onionrstatistics/serializeddata.py @@ -2,6 +2,8 @@ Serialize various node information """ +from typing import TYPE_CHECKING + from gevent import sleep from psutil import Process, WINDOWS @@ -11,6 +13,9 @@ from coredb import blockmetadb from utils.sizeutils import size, human_size from utils.identifyhome import identify_home import communicator + +if TYPE_CHECKING: + from deadsimplekv import DeadSimpleKV """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -44,18 +49,21 @@ class SerializedData: proc = Process() def get_open_files(): - if WINDOWS: return proc.num_handles() + if WINDOWS: + return proc.num_handles() return proc.num_fds() try: self._too_many except AttributeError: sleep(1) - comm_inst = self._too_many.get(communicator.OnionrCommunicatorDaemon, args=(self._too_many,)) + comm_inst = self._too_many.get(communicator.OnionrCommunicatorDaemon, + args=(self._too_many,)) kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV") connected = [] - [connected.append(x) for x in kv.get('onlinePeers') if x not in connected] - stats['uptime'] = comm_inst.getUptime() + [connected.append(x) + for x in kv.get('onlinePeers') if x not in connected] + stats['uptime'] = kv.get('getUptime') stats['connectedNodes'] = '\n'.join(connected) stats['blockCount'] = len(blockmetadb.get_block_list()) stats['blockQueueCount'] = len(kv.get('blockQueue'))