From 000538ddc8defb918bf80d079b48d11f831e692a Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Wed, 17 Jul 2019 11:25:29 -0500 Subject: [PATCH] fixes in board, refactoring core databases to not use core anymore --- .../downloadblocks/shoulddownload.py | 3 ++- onionr/communicatorutils/housekeeping.py | 3 ++- onionr/communicatorutils/lookupblocks.py | 4 ++-- onionr/communicatorutils/servicecreator.py | 4 ++-- onionr/core.py | 18 ------------------ onionr/coredb/blockmetadb/__init__.py | 13 +++++++------ onionr/coredb/dbfiles.py | 5 +++++ onionr/httpapi/miscclientapi/getblocks.py | 7 ++++--- onionr/httpapi/miscpublicapi/getblocks.py | 5 +++-- onionr/onionrblockapi.py | 8 ++++---- onionr/onionrcommands/onionrstatistics.py | 4 ++-- onionr/onionrutils/importnewblocks.py | 3 ++- onionr/serializeddata.py | 4 ++-- .../static-data/default-plugins/flow/main.py | 4 ++-- .../default-plugins/pms/loadinbox.py | 3 ++- .../static-data/default-plugins/pms/main.py | 3 ++- onionr/static-data/www/board/index.html | 19 ------------------- 17 files changed, 43 insertions(+), 67 deletions(-) create mode 100644 onionr/coredb/dbfiles.py diff --git a/onionr/communicatorutils/downloadblocks/shoulddownload.py b/onionr/communicatorutils/downloadblocks/shoulddownload.py index d126f548..525a3568 100644 --- a/onionr/communicatorutils/downloadblocks/shoulddownload.py +++ b/onionr/communicatorutils/downloadblocks/shoulddownload.py @@ -17,9 +17,10 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' +from coredb import blockmetadb def should_download(comm_inst, block_hash): ret_data = True - if block_hash in comm_inst._core.getBlockList(): # Dont download block we have + if block_hash in blockmetadb.get_block_list(): # Dont download block we have ret_data = False else: if comm_inst._core._blacklist.inBlacklist(block_hash): # Dont download blacklisted block diff --git a/onionr/communicatorutils/housekeeping.py b/onionr/communicatorutils/housekeeping.py index 3071a994..9c041f9a 100755 --- a/onionr/communicatorutils/housekeeping.py +++ b/onionr/communicatorutils/housekeeping.py @@ -21,6 +21,7 @@ import sqlite3 import logger from onionrusers import onionrusers from onionrutils import epoch +from coredb import blockmetadb def clean_old_blocks(comm_inst): '''Delete old blocks if our disk allocation is full/near full, and also expired blocks''' @@ -31,7 +32,7 @@ def clean_old_blocks(comm_inst): logger.info('Deleted block: %s' % (bHash,)) while comm_inst._core.storage_counter.isFull(): - oldest = comm_inst._core.getBlockList()[0] + oldest = blockmetadb.get_block_list()[0] comm_inst._core._blacklist.addToDB(oldest) comm_inst._core.removeBlock(oldest) logger.info('Deleted block: %s' % (oldest,)) diff --git a/onionr/communicatorutils/lookupblocks.py b/onionr/communicatorutils/lookupblocks.py index 0e36753a..c05d886d 100755 --- a/onionr/communicatorutils/lookupblocks.py +++ b/onionr/communicatorutils/lookupblocks.py @@ -20,12 +20,12 @@ import logger, onionrproofs from onionrutils import stringvalidators, epoch from communicator import peeraction, onlinepeers - +from coredb import blockmetadb def lookup_blocks_from_communicator(comm_inst): logger.info('Looking up new blocks...') tryAmount = 2 newBlocks = '' - existingBlocks = comm_inst._core.getBlockList() + existingBlocks = blockmetadb.get_block_list() triedPeers = [] # list of peers we've tried this time around maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion lastLookupTime = 0 # Last time we looked up a particular peer's list diff --git a/onionr/communicatorutils/servicecreator.py b/onionr/communicatorutils/servicecreator.py index 769f663b..15a81dbc 100755 --- a/onionr/communicatorutils/servicecreator.py +++ b/onionr/communicatorutils/servicecreator.py @@ -19,14 +19,14 @@ ''' import communicator, onionrblockapi from onionrutils import stringvalidators, bytesconverter - +from coredb import blockmetadb def service_creator(daemon): assert isinstance(daemon, communicator.OnionrCommunicatorDaemon) core = daemon._core # Find socket connection blocks # TODO cache blocks and only look at recently received ones - con_blocks = core.getBlocksByType('con') + con_blocks = blockmetadb.get_blocks_by_type('con') for b in con_blocks: if not b in daemon.active_services: bl = onionrblockapi.Block(b, core=core, decrypt=True) diff --git a/onionr/core.py b/onionr/core.py index b523c5ff..b48505b2 100755 --- a/onionr/core.py +++ b/onionr/core.py @@ -267,24 +267,6 @@ class Core: ''' return coredb.keydb.transportinfo.set_address_info(self, address, key, data) - def getBlockList(self, dateRec = None, unsaved = False): - ''' - Get list of our blocks - ''' - return coredb.blockmetadb.get_block_list(self, dateRec, unsaved) - - def getBlockDate(self, blockHash): - ''' - Returns the date a block was received - ''' - return coredb.blockmetadb.get_block_date(self, blockHash) - - def getBlocksByType(self, blockType, orderDate=True): - ''' - Returns a list of blocks by the type - ''' - return coredb.blockmetadb.get_blocks_by_type(self, blockType, orderDate) - def getExpiredBlocks(self): '''Returns a list of expired blocks''' return coredb.blockmetadb.expiredblocks.get_expired_blocks(self) diff --git a/onionr/coredb/blockmetadb/__init__.py b/onionr/coredb/blockmetadb/__init__.py index 409696dc..96ccdfaa 100644 --- a/onionr/coredb/blockmetadb/__init__.py +++ b/onionr/coredb/blockmetadb/__init__.py @@ -19,14 +19,15 @@ ''' import sqlite3 from . import expiredblocks, updateblockinfo, add -def get_block_list(core_inst, dateRec = None, unsaved = False): +from .. import dbfiles +def get_block_list(dateRec = None, unsaved = False): ''' Get list of our blocks ''' if dateRec == None: dateRec = 0 - conn = sqlite3.connect(core_inst.blockDB, timeout=30) + conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) c = conn.cursor() execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;' @@ -38,12 +39,12 @@ def get_block_list(core_inst, dateRec = None, unsaved = False): conn.close() return rows -def get_block_date(core_inst, blockHash): +def get_block_date(blockHash): ''' Returns the date a block was received ''' - conn = sqlite3.connect(core_inst.blockDB, timeout=30) + conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) c = conn.cursor() execute = 'SELECT dateReceived FROM hashes WHERE hash=?;' @@ -54,12 +55,12 @@ def get_block_date(core_inst, blockHash): conn.close() return None -def get_blocks_by_type(core_inst, blockType, orderDate=True): +def get_blocks_by_type(blockType, orderDate=True): ''' Returns a list of blocks by the type ''' - conn = sqlite3.connect(core_inst.blockDB, timeout=30) + conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30) c = conn.cursor() if orderDate: diff --git a/onionr/coredb/dbfiles.py b/onionr/coredb/dbfiles.py new file mode 100644 index 00000000..47d82316 --- /dev/null +++ b/onionr/coredb/dbfiles.py @@ -0,0 +1,5 @@ +from utils import identifyhome +home = identifyhome.identify_home() +if not home.endswith('/'): home += '/' + +block_meta_db = '%sblock-metadata.db' \ No newline at end of file diff --git a/onionr/httpapi/miscclientapi/getblocks.py b/onionr/httpapi/miscclientapi/getblocks.py index f9a6aa90..bdc7c9d8 100644 --- a/onionr/httpapi/miscclientapi/getblocks.py +++ b/onionr/httpapi/miscclientapi/getblocks.py @@ -21,6 +21,7 @@ from flask import Blueprint, Response, abort import core, onionrblockapi from httpapi import apiutils from onionrutils import stringvalidators +from coredb import blockmetadb c = core.Core() @@ -29,8 +30,8 @@ client_get_block = apiutils.GetBlockData(c) client_get_blocks = Blueprint('miscclient', __name__) @client_get_blocks.route('/getblocksbytype/') -def getBlocksByType(name): - blocks = c.getBlocksByType(name) +def get_blocks_by_type_endpoint(name): + blocks = blockmetadb.get_blocks_by_type(name) return Response(','.join(blocks)) @client_get_blocks.route('/getblockbody/') @@ -49,7 +50,7 @@ def getBlockBodyData(name): def getData(name): resp = "" if stringvalidators.validate_hash(name): - if name in c.getBlockList(): + if name in blockmetadb.get_block_list(): try: resp = client_get_block.get_block_data(name, decrypt=True) except ValueError: diff --git a/onionr/httpapi/miscpublicapi/getblocks.py b/onionr/httpapi/miscpublicapi/getblocks.py index 8fe5cbf4..c7857966 100755 --- a/onionr/httpapi/miscpublicapi/getblocks.py +++ b/onionr/httpapi/miscpublicapi/getblocks.py @@ -20,11 +20,12 @@ from flask import Response, abort import config from onionrutils import bytesconverter, stringvalidators +from coredb import blockmetadb def get_public_block_list(clientAPI, publicAPI, request): # Provide a list of our blocks, with a date offset dateAdjust = request.args.get('date') - bList = clientAPI._core.getBlockList(dateRec=dateAdjust) + bList = blockmetadb.get_block_list(dateRec=dateAdjust) if clientAPI._core.config.get('general.hide_created_blocks', True): for b in publicAPI.hideBlocks: if b in bList: @@ -37,7 +38,7 @@ def get_block_data(clientAPI, publicAPI, data): resp = '' if stringvalidators.validate_hash(data): if not clientAPI._core.config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks: - if data in clientAPI._core.getBlockList(): + if data in blockmetadb.get_block_list(): block = clientAPI.getBlockData(data, raw=True) try: block = block.encode() # Encode in case data is binary diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index 00d377b0..9fd6c005 100755 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -22,7 +22,7 @@ import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions import json, os, sys, datetime, base64, onionrstorage from onionrusers import onionrusers from onionrutils import stringvalidators, epoch - +from coredb import blockmetadb class Block: blockCacheOrder = list() # NEVER write your own code that writes to this! blockCache = dict() # should never be accessed directly, look at Block.getCache() @@ -89,7 +89,7 @@ class Block: # Check for replay attacks try: - if epoch.get_epoch() - self.core.getBlockDate(self.hash) > 60: + if epoch.get_epoch() - blockmetadb.get_block_date(self.hash) > 60: assert self.core._crypto.replayTimestampValidation(self.bmetadata['rply']) except (AssertionError, KeyError, TypeError) as e: if not self.bypassReplayCheck: @@ -180,7 +180,7 @@ class Block: self.signature = self.getHeader('sig', None) # signed data is jsonMeta + block content (no linebreak) self.signedData = (None if not self.isSigned() else self.getHeader('meta') + self.getContent()) - self.date = self.getCore().getBlockDate(self.getHash()) + self.date = blockmetadb.get_block_date(self.getHash()) self.claimedTime = self.getHeader('time', None) if not self.getDate() is None: @@ -541,7 +541,7 @@ class Block: parent = Block(hash = parent, core = core) relevant_blocks = list() - blocks = (core.getBlockList() if type is None else core.getBlocksByType(type)) + blocks = (blockmetadb.get_block_list() if type is None else blockmetadb.get_blocks_by_type(type)) for block in blocks: if Block.exists(block): diff --git a/onionr/onionrcommands/onionrstatistics.py b/onionr/onionrcommands/onionrstatistics.py index c01c5e1b..fd54516b 100755 --- a/onionr/onionrcommands/onionrstatistics.py +++ b/onionr/onionrcommands/onionrstatistics.py @@ -23,11 +23,11 @@ from onionrblockapi import Block import onionr from onionrutils import checkcommunicator, mnemonickeys from utils import sizeutils - +from coredb import blockmetadb def show_stats(o_inst): try: # define stats messages here - totalBlocks = len(o_inst.onionrCore.getBlockList()) + totalBlocks = len(blockmetadb.get_block_list()) signedBlocks = len(Block.getBlocks(signed = True)) messages = { # info about local client diff --git a/onionr/onionrutils/importnewblocks.py b/onionr/onionrutils/importnewblocks.py index e1793adf..b467f5fb 100644 --- a/onionr/onionrutils/importnewblocks.py +++ b/onionr/onionrutils/importnewblocks.py @@ -20,13 +20,14 @@ import glob import logger, core from onionrutils import blockmetadata +from coredb import blockmetadb def import_new_blocks(core_inst=None, scanDir=''): ''' This function is intended to scan for new blocks ON THE DISK and import them ''' if core_inst is None: core_inst = core.Core() - blockList = core_inst.getBlockList() + blockList = blockmetadb.get_block_list() exist = False if scanDir == '': scanDir = core_inst.blockDataLocation diff --git a/onionr/serializeddata.py b/onionr/serializeddata.py index ef58c304..8121c8d5 100755 --- a/onionr/serializeddata.py +++ b/onionr/serializeddata.py @@ -19,7 +19,7 @@ ''' import core, json - +from coredb import blockmetadb class SerializedData: def __init__(self, coreInst): ''' @@ -38,6 +38,6 @@ class SerializedData: stats = {} stats['uptime'] = self._core.onionrInst.communicatorInst.getUptime() stats['connectedNodes'] = '\n'.join(self._core.onionrInst.communicatorInst.onlinePeers) - stats['blockCount'] = len(self._core.getBlockList()) + stats['blockCount'] = len(blockmetadb.get_block_list()) stats['blockQueueCount'] = len(self._core.onionrInst.communicatorInst.blockQueue) return json.dumps(stats) diff --git a/onionr/static-data/default-plugins/flow/main.py b/onionr/static-data/default-plugins/flow/main.py index de0c60e3..16a7dc08 100755 --- a/onionr/static-data/default-plugins/flow/main.py +++ b/onionr/static-data/default-plugins/flow/main.py @@ -24,7 +24,7 @@ from onionrblockapi import Block import logger, config from onionrutils import escapeansi, epoch locale.setlocale(locale.LC_ALL, '') - +from coredb import blockmetadb sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) import flowapi # import after path insert flask_blueprint = flowapi.flask_blueprint @@ -73,7 +73,7 @@ class OnionrFlow: time.sleep(1) try: while self.flowRunning: - for block in self.myCore.getBlocksByType('txt'): + for block in blockmetadb.get_blocks_by_type('txt'): block = Block(block) if block.getMetadata('ch') != self.channel: continue diff --git a/onionr/static-data/default-plugins/pms/loadinbox.py b/onionr/static-data/default-plugins/pms/loadinbox.py index b4011b00..28ac36cf 100755 --- a/onionr/static-data/default-plugins/pms/loadinbox.py +++ b/onionr/static-data/default-plugins/pms/loadinbox.py @@ -18,13 +18,14 @@ along with this program. If not, see . ''' import onionrblockapi +from coredb import blockmetadb def load_inbox(myCore): inbox_list = [] deleted = myCore.keyStore.get('deleted_mail') if deleted is None: deleted = [] - for blockHash in myCore.getBlocksByType('pm'): + for blockHash in blockmetadb.get_blocks_by_type('pm'): block = onionrblockapi.Block(blockHash, core=myCore) block.decrypt() if block.decrypted and blockHash not in deleted: diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index 5c532877..0e01fa34 100755 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -25,6 +25,7 @@ import onionrexceptions from onionrusers import onionrusers from onionrutils import stringvalidators, escapeansi, bytesconverter import locale, sys, os, json +from coredb import blockmetadb locale.setlocale(locale.LC_ALL, '') @@ -80,7 +81,7 @@ class OnionrMail: subject = '' # this could use a lot of memory if someone has received a lot of messages - for blockHash in self.myCore.getBlocksByType('pm'): + for blockHash in blockmetadb.get_blocks_by_type('pm'): pmBlocks[blockHash] = Block(blockHash, core=self.myCore) pmBlocks[blockHash].decrypt() blockCount = 0 diff --git a/onionr/static-data/www/board/index.html b/onionr/static-data/www/board/index.html index 4d718078..f3b13bb5 100755 --- a/onionr/static-data/www/board/index.html +++ b/onionr/static-data/www/board/index.html @@ -55,27 +55,8 @@ Anonymous message board -
-
- -
-

- Identity -

-

- -

-

- Copy -

-
-
-
- - -