added storagecounter test and renamed setup
parent
311dda91d6
commit
085b90f84c
|
@ -36,7 +36,7 @@ except ModuleNotFoundError:
|
|||
|
||||
# Onionr imports
|
||||
from etc import onionrvalues # For different Onionr related constants such as versions
|
||||
import setup
|
||||
import onionrsetup as setup
|
||||
|
||||
# Ensure we have at least the minimum python version
|
||||
if sys.version_info[0] == 2 or sys.version_info[1] < onionrvalues.MIN_PY_VERSION:
|
||||
|
|
|
@ -46,7 +46,7 @@ def download_blocks_from_communicator(comm_inst):
|
|||
if not shoulddownload.should_download(comm_inst, blockHash):
|
||||
continue
|
||||
|
||||
if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.isFull():
|
||||
if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.is_full():
|
||||
# Exit loop if shutting down or offline, or disk allocation reached
|
||||
break
|
||||
# Do not download blocks being downloaded
|
||||
|
|
|
@ -43,7 +43,7 @@ def clean_old_blocks(comm_inst):
|
|||
__remove_from_upload(comm_inst, bHash)
|
||||
logger.info('Deleted block: %s' % (bHash,))
|
||||
|
||||
while comm_inst.storage_counter.isFull():
|
||||
while comm_inst.storage_counter.is_full():
|
||||
oldest = blockmetadb.get_block_list()[0]
|
||||
blacklist.addToDB(oldest)
|
||||
removeblock.remove_block(oldest)
|
||||
|
|
|
@ -40,7 +40,7 @@ def lookup_blocks_from_communicator(comm_inst):
|
|||
if not comm_inst.isOnline:
|
||||
break
|
||||
# check if disk allocation is used
|
||||
if comm_inst.storage_counter.isFull():
|
||||
if comm_inst.storage_counter.is_full():
|
||||
logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
|
||||
break
|
||||
peer = onlinepeers.pick_online_peer(comm_inst) # select random online peer
|
||||
|
|
|
@ -28,33 +28,33 @@ raw = raw.raw
|
|||
confirm = confirm.confirm
|
||||
|
||||
# debug: when there is info that could be useful for debugging purposes only
|
||||
def debug(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_DEBUG):
|
||||
def debug(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_DEBUG):
|
||||
if settings.get_level() <= level:
|
||||
log('/', data, timestamp = timestamp, prompt = prompt, terminal = terminal)
|
||||
if not error is None:
|
||||
debug('Error: ' + str(error) + parse_error())
|
||||
|
||||
# info: when there is something to notify the user of, such as the success of a process
|
||||
def info(data, timestamp = False, prompt = True, terminal = False, level = settings.LEVEL_INFO):
|
||||
def info(data: str, timestamp = False, prompt = True, terminal = False, level = settings.LEVEL_INFO):
|
||||
if settings.get_level() <= level:
|
||||
log('+', data, colors.fg.green, timestamp = timestamp, prompt = prompt, terminal = terminal)
|
||||
|
||||
# warn: when there is a potential for something bad to happen
|
||||
def warn(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_WARN):
|
||||
def warn(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_WARN):
|
||||
if not error is None:
|
||||
debug('Error: ' + str(error) + parse_error())
|
||||
if settings.get_level() <= level:
|
||||
log('!', data, colors.fg.orange, timestamp = timestamp, prompt = prompt, terminal = terminal)
|
||||
|
||||
# error: when only one function, module, or process of the program encountered a problem and must stop
|
||||
def error(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_ERROR):
|
||||
def error(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_ERROR):
|
||||
if settings.get_level() <= level:
|
||||
log('-', data, colors.fg.red, timestamp = timestamp, fd = sys.stderr, prompt = prompt, terminal = terminal)
|
||||
if not error is None:
|
||||
debug('Error: ' + str(error) + parse_error())
|
||||
|
||||
# fatal: when the something so bad has happened that the program must stop
|
||||
def fatal(data, error = None, timestamp=True, prompt = True, terminal = False, level = settings.LEVEL_FATAL):
|
||||
def fatal(data: str, error = None, timestamp=True, prompt = True, terminal = False, level = settings.LEVEL_FATAL):
|
||||
if not error is None:
|
||||
debug('Error: ' + str(error) + parse_error(), terminal = terminal)
|
||||
if settings.get_level() <= level:
|
||||
|
|
|
@ -64,7 +64,7 @@ def set_level(level):
|
|||
global _level
|
||||
_level = level
|
||||
|
||||
def get_level():
|
||||
def get_level()->int:
|
||||
'''
|
||||
Get the lowest log level currently being outputted
|
||||
'''
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
from typing import Union
|
||||
import json
|
||||
from onionrutils import bytesconverter, epoch
|
||||
import storagecounter, filepaths, onionrstorage
|
||||
|
@ -8,17 +9,21 @@ from onionrusers import onionrusers
|
|||
from onionrutils import localcommand, blockmetadata, stringvalidators
|
||||
import coredb
|
||||
import onionrproofs
|
||||
def insert_block(data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False):
|
||||
'''
|
||||
import logger
|
||||
def insert_block(data: Union[str, bytes], header: str ='txt',
|
||||
sign: bool =False, encryptType:str ='', symKey:str ='',
|
||||
asymPeer:str ='', meta:dict = {},
|
||||
expire:Union[int, None] =None, disableForward:bool =False)->Union[str,bool]:
|
||||
"""
|
||||
Inserts a block into the network
|
||||
encryptType must be specified to encrypt a block
|
||||
'''
|
||||
"""
|
||||
use_subprocess = powchoice.use_subprocess(config)
|
||||
storage_counter = storagecounter.StorageCounter()
|
||||
allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
|
||||
if storage_counter.isFull():
|
||||
if storage_counter.is_full():
|
||||
logger.error(allocationReachedMessage)
|
||||
return False
|
||||
raise onionrexceptions.DiskAllocationReached
|
||||
retData = False
|
||||
|
||||
if type(data) is None:
|
||||
|
|
|
@ -1,9 +1,9 @@
|
|||
'''
|
||||
Onionr - P2P Anonymous Storage Network
|
||||
"""
|
||||
Onionr - Private P2P Communication
|
||||
|
||||
This file contains exceptions for onionr
|
||||
'''
|
||||
'''
|
||||
"""
|
||||
"""
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
|
@ -16,7 +16,7 @@
|
|||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
"""
|
||||
|
||||
# general exceptions
|
||||
class NotFound(Exception):
|
||||
|
@ -67,11 +67,11 @@ class NoDataAvailable(Exception):
|
|||
pass
|
||||
|
||||
class InvalidHexHash(Exception):
|
||||
'''When a string is not a valid hex string of appropriate length for a hash value'''
|
||||
"""When a string is not a valid hex string of appropriate length for a hash value"""
|
||||
pass
|
||||
|
||||
class InvalidProof(Exception):
|
||||
'''When a proof is invalid or inadequate'''
|
||||
"""When a proof is invalid or inadequate"""
|
||||
pass
|
||||
|
||||
# network level exceptions
|
||||
|
@ -100,4 +100,4 @@ class MissingAddress(Exception):
|
|||
# Contact exceptions
|
||||
|
||||
class ContactDeleted(Exception):
|
||||
pass
|
||||
pass
|
||||
|
|
|
@ -27,7 +27,7 @@ def getDifficultyModifier():
|
|||
on a variety of factors, currently only disk use.
|
||||
'''
|
||||
retData = 0
|
||||
useFunc = storagecounter.StorageCounter().getPercent
|
||||
useFunc = storagecounter.StorageCounter().get_percent
|
||||
|
||||
percentUse = useFunc()
|
||||
|
||||
|
|
|
@ -22,7 +22,7 @@ import config, logger, netcontroller
|
|||
from etc import onionrvalues
|
||||
from logger.settings import *
|
||||
|
||||
def setup_config(o_inst = None):
|
||||
def setup_config():
|
||||
config.reload()
|
||||
|
||||
if not os.path.exists(config._configfile):
|
||||
|
@ -42,14 +42,6 @@ def setup_config(o_inst = None):
|
|||
settings = settings | OUTPUT_TO_FILE
|
||||
set_settings(settings)
|
||||
|
||||
if not o_inst is None:
|
||||
if str(config.get('general.dev_mode', True)).lower() == 'true':
|
||||
o_inst._developmentMode = True
|
||||
set_level(LEVEL_DEBUG)
|
||||
else:
|
||||
o_inst._developmentMode = False
|
||||
set_level(LEVEL_INFO)
|
||||
|
||||
verbosity = str(config.get('log.verbosity', 'default')).lower().strip()
|
||||
if not verbosity in ['default', 'null', 'none', 'nil']:
|
||||
map = {
|
|
@ -18,6 +18,6 @@ def remove_block(block):
|
|||
conn.commit()
|
||||
conn.close()
|
||||
dataSize = sys.getsizeof(onionrstorage.getData(block))
|
||||
storagecounter.StorageCounter().removeBytes(dataSize)
|
||||
storagecounter.StorageCounter().remove_bytes(dataSize)
|
||||
else:
|
||||
raise onionrexceptions.InvalidHexHash
|
|
@ -3,7 +3,7 @@ import onionrstorage, onionrexceptions, onionrcrypto as crypto
|
|||
import filepaths, storagecounter
|
||||
from coredb import dbfiles
|
||||
from onionrutils import blockmetadata, bytesconverter
|
||||
def set_data(data):
|
||||
def set_data(data)->str:
|
||||
'''
|
||||
Set the data assciated with a hash
|
||||
'''
|
||||
|
@ -24,7 +24,7 @@ def set_data(data):
|
|||
try:
|
||||
onionrstorage.getData(dataHash)
|
||||
except onionrexceptions.NoDataAvailable:
|
||||
if storage_counter.addBytes(dataSize) != False:
|
||||
if storage_counter.add_bytes(dataSize) != False:
|
||||
onionrstorage.store(data, blockHash=dataHash)
|
||||
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
|
||||
c = conn.cursor()
|
||||
|
|
|
@ -1,9 +1,9 @@
|
|||
'''
|
||||
"""
|
||||
Onionr - Private P2P Communication
|
||||
|
||||
Keeps track of how much disk space we're using
|
||||
'''
|
||||
'''
|
||||
"""
|
||||
"""
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
|
@ -16,52 +16,54 @@
|
|||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
"""
|
||||
import config, filepaths
|
||||
config.reload()
|
||||
class StorageCounter:
|
||||
def __init__(self):
|
||||
self.dataFile = filepaths.usage_file
|
||||
self.data_file = filepaths.usage_file
|
||||
return
|
||||
|
||||
def isFull(self):
|
||||
retData = False
|
||||
if config.get('allocations.disk', 2000000000) <= (self.getAmount() + 1000):
|
||||
retData = True
|
||||
return retData
|
||||
def is_full(self)->bool:
|
||||
"""Returns if the allocated disk space is full (this is Onionr config, not true FS capacity)"""
|
||||
ret_data = False
|
||||
if config.get('allocations.disk', 2000000000) <= (self.get_amount() + 1000):
|
||||
ret_data = True
|
||||
return ret_data
|
||||
|
||||
def _update(self, data):
|
||||
with open(self.dataFile, 'w') as dataFile:
|
||||
dataFile.write(str(data))
|
||||
def getAmount(self):
|
||||
'''Return how much disk space we're using (according to record)'''
|
||||
retData = 0
|
||||
with open(self.data_file, 'w') as data_file:
|
||||
data_file.write(str(data))
|
||||
|
||||
def get_amount(self)->int:
|
||||
"""Return how much disk space we're using (according to record)"""
|
||||
ret_data = 0
|
||||
try:
|
||||
with open(self.dataFile, 'r') as dataFile:
|
||||
retData = int(dataFile.read())
|
||||
with open(self.data_file, 'r') as data_file:
|
||||
ret_data = int(data_file.read())
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except ValueError:
|
||||
pass # Possibly happens when the file is empty
|
||||
return retData
|
||||
return ret_data
|
||||
|
||||
def getPercent(self):
|
||||
'''Return percent (decimal/float) of disk space we're using'''
|
||||
amount = self.getAmount()
|
||||
def get_percent(self)->int:
|
||||
"""Return percent (decimal/float) of disk space we're using"""
|
||||
amount = self.get_amount()
|
||||
return round(amount / config.get('allocations.disk', 2000000000), 2)
|
||||
|
||||
def addBytes(self, amount):
|
||||
'''Record that we are now using more disk space, unless doing so would exceed configured max'''
|
||||
newAmount = amount + self.getAmount()
|
||||
retData = newAmount
|
||||
if newAmount > config.get('allocations.disk', 2000000000):
|
||||
retData = False
|
||||
def add_bytes(self, amount)->int:
|
||||
"""Record that we are now using more disk space, unless doing so would exceed configured max"""
|
||||
new_amount = amount + self.get_amount()
|
||||
ret_data = new_amount
|
||||
if new_amount > config.get('allocations.disk', 2000000000):
|
||||
ret_data = False
|
||||
else:
|
||||
self._update(newAmount)
|
||||
return retData
|
||||
self._update(new_amount)
|
||||
return ret_data
|
||||
|
||||
def removeBytes(self, amount):
|
||||
'''Record that we are now using less disk space'''
|
||||
newAmount = self.getAmount() - amount
|
||||
self._update(newAmount)
|
||||
return newAmount
|
||||
def remove_bytes(self, amount)->int:
|
||||
"""Record that we are now using less disk space"""
|
||||
new_amount = self.get_amount() - amount
|
||||
self._update(new_amount)
|
||||
return new_amount
|
||||
|
|
|
@ -8,7 +8,7 @@ print("Test directory:", TEST_DIR)
|
|||
os.environ["ONIONR_HOME"] = TEST_DIR
|
||||
from utils import createdirs
|
||||
from coredb import keydb
|
||||
import setup, keymanager, filepaths
|
||||
import onionrsetup as setup, keymanager, filepaths
|
||||
from onionrutils import stringvalidators
|
||||
createdirs.create_dirs()
|
||||
setup.setup_config()
|
||||
|
|
|
@ -8,7 +8,7 @@ print("Test directory:", TEST_DIR)
|
|||
os.environ["ONIONR_HOME"] = TEST_DIR
|
||||
from utils import networkmerger, createdirs
|
||||
from coredb import keydb
|
||||
import setup
|
||||
import onionrsetup as setup
|
||||
from utils import createdirs
|
||||
createdirs.create_dirs()
|
||||
setup.setup_config()
|
||||
|
|
|
@ -0,0 +1,45 @@
|
|||
import sys, os
|
||||
sys.path.append(".")
|
||||
import unittest, uuid
|
||||
|
||||
import logger
|
||||
import config
|
||||
from utils import createdirs
|
||||
import onionrsetup as setup
|
||||
from utils import createdirs
|
||||
import onionrblocks
|
||||
import filepaths
|
||||
import onionrexceptions
|
||||
import storagecounter
|
||||
import onionrstorage
|
||||
|
||||
def _test_setup():
|
||||
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
|
||||
print("Test directory:", TEST_DIR)
|
||||
os.environ["ONIONR_HOME"] = TEST_DIR
|
||||
createdirs.create_dirs()
|
||||
setup.setup_config()
|
||||
config.reload()
|
||||
|
||||
class TestStorageCounter(unittest.TestCase):
|
||||
def test_basic_amount(self):
|
||||
_test_setup()
|
||||
self.assertIsNotNone(config.get('allocations.disk'))
|
||||
self.assertGreater(config.get('allocations.disk'), 1000000)
|
||||
|
||||
def test_insert_too_much(self):
|
||||
_test_setup()
|
||||
config.set('allocations.disk', 1000)
|
||||
self.assertRaises(onionrexceptions.DiskAllocationReached, onionrblocks.insert, "test")
|
||||
|
||||
def test_count(self):
|
||||
_test_setup()
|
||||
counter = storagecounter.StorageCounter()
|
||||
start_value = counter.get_amount()
|
||||
b_hash = onionrblocks.insert("test")
|
||||
self.assertGreater(counter.get_amount(), start_value)
|
||||
onionrstorage.removeblock.remove_block(b_hash)
|
||||
self.assertEqual(counter.get_amount(), start_value)
|
||||
|
||||
|
||||
unittest.main()
|
|
@ -5,9 +5,9 @@ gevent==1.3.6
|
|||
Flask==1.1.1
|
||||
PySocks==1.6.8
|
||||
stem==1.7.1
|
||||
deadsimplekv==0.1.1
|
||||
deadsimplekv==0.2.0
|
||||
unpaddedbase32==0.1.0
|
||||
streamedrequests==1.0.0
|
||||
jinja2==2.10.1
|
||||
toomanyobjs==1.1.0
|
||||
mnemonic==0.18
|
||||
mnemonic==0.18
|
||||
|
|
|
@ -46,8 +46,8 @@ click==7.0 \
|
|||
--hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
|
||||
--hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
|
||||
# via flask
|
||||
deadsimplekv==0.1.1 \
|
||||
--hash=sha256:4bf951e188c302006e37f95bde6117b1b938fb454153d583c6346090d9bead1a
|
||||
deadsimplekv==0.2.0 \
|
||||
--hash=sha256:81405408a4d23cc94ac359f9570e0ff198b67e5a93e3ae32eca85e3b62252f38
|
||||
flask==1.1.1 \
|
||||
--hash=sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52 \
|
||||
--hash=sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6
|
||||
|
|
Loading…
Reference in New Issue