added storagecounter test and renamed setup

onionr/__init__.py

@@ -36,7 +36,7 @@ except ModuleNotFoundError:
 
 # Onionr imports
 from etc import onionrvalues # For different Onionr related constants such as versions
-import setup
+import onionrsetup as setup
 
 # Ensure we have at least the minimum python version
 if sys.version_info[0] == 2 or sys.version_info[1] < onionrvalues.MIN_PY_VERSION:

onionr/communicatorutils/downloadblocks/__init__.py

@@ -46,7 +46,7 @@ def download_blocks_from_communicator(comm_inst):
         if not shoulddownload.should_download(comm_inst, blockHash):
             continue
 
-        if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.isFull():
+        if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.is_full():
             # Exit loop if shutting down or offline, or disk allocation reached
             break
         # Do not download blocks being downloaded

onionr/communicatorutils/housekeeping.py

@@ -43,7 +43,7 @@ def clean_old_blocks(comm_inst):
         __remove_from_upload(comm_inst, bHash)
         logger.info('Deleted block: %s' % (bHash,))
 
-    while comm_inst.storage_counter.isFull():
+    while comm_inst.storage_counter.is_full():
         oldest = blockmetadb.get_block_list()[0]
         blacklist.addToDB(oldest)
         removeblock.remove_block(oldest)

onionr/communicatorutils/lookupblocks.py

@@ -40,7 +40,7 @@ def lookup_blocks_from_communicator(comm_inst):
         if not comm_inst.isOnline:
             break
         # check if disk allocation is used
-        if comm_inst.storage_counter.isFull():
+        if comm_inst.storage_counter.is_full():
             logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
             break
         peer = onlinepeers.pick_online_peer(comm_inst) # select random online peer

onionr/logger/__init__.py

@@ -28,33 +28,33 @@ raw = raw.raw
 confirm = confirm.confirm
 
 # debug: when there is info that could be useful for debugging purposes only
-def debug(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_DEBUG):
+def debug(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_DEBUG):
     if settings.get_level() <= level:
         log('/', data, timestamp = timestamp, prompt = prompt, terminal = terminal)
     if not error is None:
         debug('Error: ' + str(error) + parse_error())
 
 # info: when there is something to notify the user of, such as the success of a process
-def info(data, timestamp = False, prompt = True, terminal = False, level = settings.LEVEL_INFO):
+def info(data: str, timestamp = False, prompt = True, terminal = False, level = settings.LEVEL_INFO):
     if settings.get_level() <= level:
         log('+', data, colors.fg.green, timestamp = timestamp, prompt = prompt, terminal = terminal)
 
 # warn: when there is a potential for something bad to happen
-def warn(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_WARN):
+def warn(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_WARN):
     if not error is None:
         debug('Error: ' + str(error) + parse_error())
     if settings.get_level() <= level:
         log('!', data, colors.fg.orange, timestamp = timestamp, prompt = prompt, terminal = terminal)
 
 # error: when only one function, module, or process of the program encountered a problem and must stop
-def error(data, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_ERROR):
+def error(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_ERROR):
     if settings.get_level() <= level:
         log('-', data, colors.fg.red, timestamp = timestamp, fd = sys.stderr, prompt = prompt, terminal = terminal)
     if not error is None:
         debug('Error: ' + str(error) + parse_error())
 
 # fatal: when the something so bad has happened that the program must stop
-def fatal(data, error = None, timestamp=True, prompt = True, terminal = False, level = settings.LEVEL_FATAL):
+def fatal(data: str, error = None, timestamp=True, prompt = True, terminal = False, level = settings.LEVEL_FATAL):
     if not error is None:
         debug('Error: ' + str(error) + parse_error(), terminal = terminal)
     if settings.get_level() <= level:

onionr/logger/settings.py

@@ -64,7 +64,7 @@ def set_level(level):
     global _level
     _level = level
 
-def get_level():
+def get_level()->int:
     '''
         Get the lowest log level currently being outputted
     '''

onionr/onionrblocks/insert.py

@@ -1,3 +1,4 @@
+from typing import Union
 import json
 from onionrutils import bytesconverter, epoch
 import storagecounter, filepaths, onionrstorage
@@ -8,17 +9,21 @@ from onionrusers import onionrusers
 from onionrutils import localcommand, blockmetadata, stringvalidators
 import coredb
 import onionrproofs
-def insert_block(data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False):
+import logger
-    '''
+def insert_block(data: Union[str, bytes], header: str ='txt', 
+                sign: bool =False, encryptType:str ='', symKey:str ='',
+                asymPeer:str ='', meta:dict = {},
+                expire:Union[int, None] =None, disableForward:bool =False)->Union[str,bool]:
+    """
         Inserts a block into the network
         encryptType must be specified to encrypt a block
-    '''
+    """
     use_subprocess = powchoice.use_subprocess(config)
     storage_counter = storagecounter.StorageCounter()
     allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
-    if storage_counter.isFull():
+    if storage_counter.is_full():
         logger.error(allocationReachedMessage)
-        return False
+        raise onionrexceptions.DiskAllocationReached
     retData = False
 
     if type(data) is None:

onionr/onionrexceptions.py

@@ -1,9 +1,9 @@
-'''
+"""
-    Onionr - P2P Anonymous Storage Network
+    Onionr - Private P2P Communication
 
     This file contains exceptions for onionr
-'''
+"""
-'''
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,7 +16,7 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
 
 # general exceptions
 class NotFound(Exception):
@@ -67,11 +67,11 @@ class NoDataAvailable(Exception):
     pass
 
 class InvalidHexHash(Exception):
-    '''When a string is not a valid hex string of appropriate length for a hash value'''
+    """When a string is not a valid hex string of appropriate length for a hash value"""
     pass
 
 class InvalidProof(Exception):
-    '''When a proof is invalid or inadequate'''
+    """When a proof is invalid or inadequate"""
     pass
 
 # network level exceptions
@@ -100,4 +100,4 @@ class MissingAddress(Exception):
 # Contact exceptions
 
 class ContactDeleted(Exception):
-    pass
+    pass

onionr/onionrproofs.py

@@ -27,7 +27,7 @@ def getDifficultyModifier():
     on a variety of factors, currently only disk use.
     '''
     retData = 0
-    useFunc = storagecounter.StorageCounter().getPercent
+    useFunc = storagecounter.StorageCounter().get_percent
 
     percentUse = useFunc()
 

onionr/onionrsetup/setupconfig.py

@@ -22,7 +22,7 @@ import config, logger, netcontroller
 from etc import onionrvalues
 from logger.settings import *
 
-def setup_config(o_inst = None):
+def setup_config():
     config.reload()
     
     if not os.path.exists(config._configfile):
@@ -42,14 +42,6 @@ def setup_config(o_inst = None):
         settings = settings | OUTPUT_TO_FILE
     set_settings(settings)
 
-    if not o_inst is None:
-        if str(config.get('general.dev_mode', True)).lower() == 'true':
-            o_inst._developmentMode = True
-            set_level(LEVEL_DEBUG)
-        else:
-            o_inst._developmentMode = False
-            set_level(LEVEL_INFO)
-
     verbosity = str(config.get('log.verbosity', 'default')).lower().strip()
     if not verbosity in ['default', 'null', 'none', 'nil']:
         map = {

onionr/onionrstorage/removeblock.py

@@ -18,6 +18,6 @@ def remove_block(block):
         conn.commit()
         conn.close()
         dataSize = sys.getsizeof(onionrstorage.getData(block))
-        storagecounter.StorageCounter().removeBytes(dataSize)
+        storagecounter.StorageCounter().remove_bytes(dataSize)
     else:
         raise onionrexceptions.InvalidHexHash

onionr/onionrstorage/setdata.py

@@ -3,7 +3,7 @@ import onionrstorage, onionrexceptions, onionrcrypto as crypto
 import filepaths, storagecounter
 from coredb import dbfiles
 from onionrutils import blockmetadata, bytesconverter
-def set_data(data):
+def set_data(data)->str:
     '''
         Set the data assciated with a hash
     '''
@@ -24,7 +24,7 @@ def set_data(data):
     try:
         onionrstorage.getData(dataHash)
     except onionrexceptions.NoDataAvailable:
-        if storage_counter.addBytes(dataSize) != False:
+        if storage_counter.add_bytes(dataSize) != False:
             onionrstorage.store(data, blockHash=dataHash)
             conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
             c = conn.cursor()

onionr/storagecounter.py

@@ -1,9 +1,9 @@
-'''
+"""
     Onionr - Private P2P Communication
 
     Keeps track of how much disk space we're using
-'''
+"""
-'''
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,52 +16,54 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
 import config, filepaths
 config.reload()
 class StorageCounter:
     def __init__(self):
-        self.dataFile = filepaths.usage_file
+        self.data_file = filepaths.usage_file
         return
 
-    def isFull(self):
+    def is_full(self)->bool:
-        retData = False
+        """Returns if the allocated disk space is full (this is Onionr config, not true FS capacity)"""
-        if config.get('allocations.disk', 2000000000) <= (self.getAmount() + 1000):
+        ret_data = False
-            retData = True
+        if config.get('allocations.disk', 2000000000) <= (self.get_amount() + 1000):
-        return retData
+            ret_data = True
+        return ret_data
 
     def _update(self, data):
-        with open(self.dataFile, 'w') as dataFile:
+        with open(self.data_file, 'w') as data_file:
-            dataFile.write(str(data))
+            data_file.write(str(data))
-    def getAmount(self):
+
-        '''Return how much disk space we're using (according to record)'''
+    def get_amount(self)->int:
-        retData = 0
+        """Return how much disk space we're using (according to record)"""
+        ret_data = 0
         try:
-            with open(self.dataFile, 'r') as dataFile:
+            with open(self.data_file, 'r') as data_file:
-                retData = int(dataFile.read())
+                ret_data = int(data_file.read())
         except FileNotFoundError:
             pass
         except ValueError:
             pass # Possibly happens when the file is empty
-        return retData
+        return ret_data
     
-    def getPercent(self):
+    def get_percent(self)->int:
-        '''Return percent (decimal/float) of disk space we're using'''
+        """Return percent (decimal/float) of disk space we're using"""
-        amount = self.getAmount()
+        amount = self.get_amount()
         return round(amount / config.get('allocations.disk', 2000000000), 2)
 
-    def addBytes(self, amount):
+    def add_bytes(self, amount)->int:
-        '''Record that we are now using more disk space, unless doing so would exceed configured max'''
+        """Record that we are now using more disk space, unless doing so would exceed configured max"""
-        newAmount = amount + self.getAmount()
+        new_amount = amount + self.get_amount()
-        retData = newAmount
+        ret_data = new_amount
-        if newAmount > config.get('allocations.disk', 2000000000):
+        if new_amount > config.get('allocations.disk', 2000000000):
-            retData = False
+            ret_data = False
         else:
-            self._update(newAmount)
+            self._update(new_amount)
-        return retData
+        return ret_data
 
-    def removeBytes(self, amount):
+    def remove_bytes(self, amount)->int:
-        '''Record that we are now using less disk space'''
+        """Record that we are now using less disk space"""
-        newAmount = self.getAmount() - amount
+        new_amount = self.get_amount() - amount
-        self._update(newAmount)
+        self._update(new_amount)
-        return newAmount
+        return new_amount

onionr/tests/test_keymanager.py

@@ -8,7 +8,7 @@ print("Test directory:", TEST_DIR)
 os.environ["ONIONR_HOME"] = TEST_DIR
 from utils import createdirs
 from coredb import keydb
-import setup, keymanager, filepaths
+import onionrsetup as setup, keymanager, filepaths
 from onionrutils import stringvalidators
 createdirs.create_dirs()
 setup.setup_config()

onionr/tests/test_networkmerger.py

@@ -8,7 +8,7 @@ print("Test directory:", TEST_DIR)
 os.environ["ONIONR_HOME"] = TEST_DIR
 from utils import networkmerger, createdirs
 from coredb import keydb
-import setup
+import onionrsetup as setup
 from utils import createdirs
 createdirs.create_dirs()
 setup.setup_config()

onionr/tests/test_storagecounter.py

@@ -0,0 +1,45 @@
+import sys, os
+sys.path.append(".")
+import unittest, uuid
+
+import logger
+import config
+from utils import createdirs
+import onionrsetup as setup
+from utils import createdirs
+import onionrblocks
+import filepaths
+import onionrexceptions
+import storagecounter
+import onionrstorage
+
+def _test_setup():
+    TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
+    print("Test directory:", TEST_DIR)
+    os.environ["ONIONR_HOME"] = TEST_DIR
+    createdirs.create_dirs()
+    setup.setup_config()
+    config.reload()
+
+class TestStorageCounter(unittest.TestCase):
+    def test_basic_amount(self):
+        _test_setup()
+        self.assertIsNotNone(config.get('allocations.disk'))
+        self.assertGreater(config.get('allocations.disk'), 1000000)
+    
+    def test_insert_too_much(self):
+        _test_setup()
+        config.set('allocations.disk', 1000)
+        self.assertRaises(onionrexceptions.DiskAllocationReached, onionrblocks.insert, "test")
+
+    def test_count(self):
+        _test_setup()
+        counter = storagecounter.StorageCounter()
+        start_value = counter.get_amount()
+        b_hash = onionrblocks.insert("test")
+        self.assertGreater(counter.get_amount(), start_value)
+        onionrstorage.removeblock.remove_block(b_hash)
+        self.assertEqual(counter.get_amount(), start_value)
+    
+
+unittest.main()

requirements.in

@@ -5,9 +5,9 @@ gevent==1.3.6
 Flask==1.1.1
 PySocks==1.6.8
 stem==1.7.1
-deadsimplekv==0.1.1
+deadsimplekv==0.2.0
 unpaddedbase32==0.1.0
 streamedrequests==1.0.0
 jinja2==2.10.1
 toomanyobjs==1.1.0
-mnemonic==0.18
+mnemonic==0.18

requirements.txt

@@ -46,8 +46,8 @@ click==7.0 \
     --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
     --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
     # via flask
-deadsimplekv==0.1.1 \
+deadsimplekv==0.2.0 \
-    --hash=sha256:4bf951e188c302006e37f95bde6117b1b938fb454153d583c6346090d9bead1a
+    --hash=sha256:81405408a4d23cc94ac359f9570e0ff198b67e5a93e3ae32eca85e3b62252f38
 flask==1.1.1 \
     --hash=sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52 \
     --hash=sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6