dxs/onionr
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Moved blockQueue to DSKV singleton as part of communicator decoupling

Browse source
This commit is contained in:
Kevin 2020-07-24 14:37:01 -05:00
parent 47013431d2
commit 6ecb62356a
6 changed files with 37 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

21
src/communicatorutils/downloadblocks/__init__.py
View file

@ -6,6 +6,7 @@
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from communicator import OnionrCommunicatorDaemon
from deadsimplekv import DeadSimpleKV
from gevent import spawn
@ -45,15 +46,16 @@ storage_counter = storagecounter.StorageCounter()
def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
"""Use communicator instance to download blocks in the comms's queue"""
blacklist = onionrblacklist.OnionrBlackList()
kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
count: int = 0
metadata_validation_result: bool = False
# Iterate the block queue in the communicator
for blockHash in list(comm_inst.blockQueue):
for blockHash in list(kv.get('blockQueue')):
count += 1
try:
blockPeers = list(comm_inst.blockQueue[blockHash])
blockPeers = list(kv.get('blockQueue')[blockHash])
except KeyError:
blockPeers = []
removeFromQueue = True
@ -61,7 +63,8 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
if not shoulddownload.should_download(comm_inst, blockHash):
continue
if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.is_full():
if comm_inst.shutdown or not comm_inst.isOnline or \
storage_counter.is_full():
# Exit loop if shutting down or offline, or disk allocation reached
break
# Do not download blocks being downloaded
@ -82,8 +85,12 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
peerUsed = blockPeers.pop(0)
if not comm_inst.shutdown and peerUsed.strip() != '':
logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
content = peeraction.peer_action(comm_inst, peerUsed, 'getdata/' + blockHash, max_resp_size=3000000) # block content from random peer (includes metadata)
logger.info(
f"Attempting to download %s from {peerUsed}..." % (blockHash[:12],))
content = peeraction.peer_action(
comm_inst, peerUsed,
'getdata/' + blockHash,
max_resp_size=3000000) # block content from random peer
if content is not False and len(content) > 0:
try:
@ -151,10 +158,10 @@ def download_blocks_from_communicator(comm_inst: "OnionrCommunicatorDaemon"):
removeFromQueue = False # Don't remove from queue if 404
if removeFromQueue:
try:
del comm_inst.blockQueue[blockHash] # remove from block queue both if success or false
del kv.get('blockQueue')[blockHash] # remove from block queue both if success or false
if count == LOG_SKIP_COUNT:
logger.info('%s blocks remaining in queue' %
[len(comm_inst.blockQueue)], terminal=True)
[len(kv.get('blockQueue'))], terminal=True)
count = 0
except KeyError:
pass

3
src/communicatorutils/downloadblocks/shoulddownload.py
View file

@ -25,6 +25,7 @@ def should_download(comm_inst, block_hash) -> bool:
"""Return bool for if a (assumed to exist) block should be downloaded."""
blacklist = onionrblacklist.OnionrBlackList()
should = True
kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
if block_hash in blockmetadb.get_block_list():
# Don't download block we have
should = False
@ -35,7 +36,7 @@ def should_download(comm_inst, block_hash) -> bool:
if should is False:
# Remove block from communicator queue if it shouldn't be downloaded
try:
del comm_inst.blockQueue[block_hash]
del kv.get('blockQueue')[block_hash]
except KeyError:
pass
return should

15
src/communicatorutils/lookupblocks.py
View file

@ -45,10 +45,11 @@ def lookup_blocks_from_communicator(comm_inst):
maxBacklog = 1560
lastLookupTime = 0 # Last time we looked up a particular peer's list
new_block_count = 0
kv: "DeadSimpleKV" = comm_inst.shared_state.get_by_string("DeadSimpleKV")
for i in range(tryAmount):
# Defined here to reset it each time, time offset is added later
listLookupCommand = 'getblocklist'
if len(comm_inst.blockQueue) >= maxBacklog:
if len(kv.get('blockQueue')) >= maxBacklog:
break
if not comm_inst.isOnline:
break
@ -100,19 +101,19 @@ def lookup_blocks_from_communicator(comm_inst):
# if block does not exist on disk + is not already in queue
if i not in existingBlocks:
if i not in comm_inst.blockQueue:
if i not in kv.get('blockQueue'):
if onionrproofs.hashMeetsDifficulty(i) and \
not blacklist.inBlacklist(i):
if len(comm_inst.blockQueue) <= 1000000:
if len(kv.get('blockQueue')) <= 1000000:
# add blocks to download queue
comm_inst.blockQueue[i] = [peer]
kv.get('blockQueue')[i] = [peer]
new_block_count += 1
comm_inst.dbTimestamps[peer] = \
epoch.get_rounded_epoch(roundS=60)
else:
if peer not in comm_inst.blockQueue[i]:
if len(comm_inst.blockQueue[i]) < 10:
comm_inst.blockQueue[i].append(peer)
if peer not in kv.get('blockQueue')[i]:
if len(kv.get('blockQueue')[i]) < 10:
kv.get('blockQueue')[i].append(peer)
if new_block_count > 0:
block_string = ""
if new_block_count > 1: