merge gitlab

onionr/api.py

@@ -52,7 +52,7 @@ class API:
 
         config.reload()
 
-        if config.get('devmode', True):
+        if config.get('dev_mode', True):
             self._developmentMode = True
             logger.set_level(logger.LEVEL_DEBUG)
         else:
@@ -65,12 +65,12 @@ class API:
         self._crypto = onionrcrypto.OnionrCrypto(self._core)
         self._utils = onionrutils.OnionrUtils(self._core)
         app = flask.Flask(__name__)
-        bindPort = int(config.get('client')['port'])
+        bindPort = int(config.get('client.port', 59496))
         self.bindPort = bindPort
-        self.clientToken = config.get('client')['client_hmac']
+        self.clientToken = config.get('client.hmac')
         self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
 
-        self.i2pEnabled = config.get('i2p', {'host' : False})['host']
+        self.i2pEnabled = config.get('i2p.host', False)
 
         self.mimeType = 'text/plain'
 
@@ -82,7 +82,7 @@ class API:
             self.host = '.'.join(hostOctets)
         else:
             self.host = '127.0.0.1'
-        
+
         with open('data/host.txt', 'w') as file:
             file.write(self.host)
 

onionr/communicator.py

@@ -75,7 +75,7 @@ class OnionrCommunicate:
         plugins.reload()
 
         # Print nice header thing :)
-        if config.get('display_header', True):
+        if config.get('general.display_header', True):
             self.header()
 
         while True:
@@ -149,7 +149,7 @@ class OnionrCommunicate:
 
                         logger.info('Checking for callbacks with connection %s...' % data['id'])
 
-                        self.check_callbacks(data, config.get('dc_execcallbacks', True))
+                        self.check_callbacks(data, config.get('general.dc_execcallbacks', True))
 
                         events.event('incoming_direct_connection', data = {'callback' : True, 'communicator' : self, 'data' : data})
                     except Exception as e:
@@ -349,7 +349,7 @@ class OnionrCommunicate:
             If yet another callback is requested, it can be put in the `callback` parameter.
         '''
 
-        if config.get('dc_response', True):
+        if config.get('general.dc_response', True):
             data['id'] = identifier
             data['sender'] = open('data/hs/hostname').read()
             data['callback'] = True
@@ -768,7 +768,7 @@ class OnionrCommunicate:
 shouldRun = False
 debug = True
 developmentMode = False
-if config.get('devmode', True):
+if config.get('general.dev_mode', True):
     developmentMode = True
 try:
     if sys.argv[1] == 'run':

onionr/communicator2.py

@@ -25,6 +25,8 @@ from defusedxml import minidom
 
 class OnionrCommunicatorDaemon:
     def __init__(self, debug, developmentMode):
+        logger.warn('New (unstable) communicator is being used.')
+
         self.timers = []
         self._core = core.Core()
         self.nistSaltTimestamp = 0
@@ -48,7 +50,7 @@ class OnionrCommunicatorDaemon:
         plugins.reload()
 
         # Print nice header thing :)
-        if config.get('display_header', True):
+        if config.get('general.display_header', True):
             self.header()
 
         if debug or developmentMode:
@@ -217,7 +219,7 @@ class OnionrCommunicatorDaemon:
                 # only to stdout, not file or log or anything
                 print(file.read().decode().replace('P', logger.colors.fg.pink).replace('W', logger.colors.reset + logger.colors.bold).replace('G', logger.colors.fg.green).replace('\n', logger.colors.reset + '\n'))
                 logger.info(logger.colors.fg.lightgreen + '-> ' + str(message) + logger.colors.reset + logger.colors.fg.lightgreen + ' <-\n')
-    
+
 class OnionrCommunicatorTimers:
     def __init__(self, daemonInstance, timerFunction, frequency, makeThread=True, threadAmount=1, maxThreads=5):
         self.timerFunction = timerFunction
@@ -255,7 +257,7 @@ class OnionrCommunicatorTimers:
 shouldRun = False
 debug = True
 developmentMode = False
-if config.get('devmode', True):
+if config.get('general.dev_mode', True):
     developmentMode = True
 try:
     if sys.argv[1] == 'run':
@@ -269,4 +271,4 @@ if shouldRun:
         sys.exit(1)
         pass
     except Exception as e:
-        logger.error('Error occured in Communicator', error = e, timestamp = False)
+        logger.error('Error occured in Communicator', error = e, timestamp = False)

onionr/config.py

@@ -28,9 +28,20 @@ def get(key, default = None):
         Gets the key from configuration, or returns `default`
     '''
 
-    if is_set(key):
+    key = str(key).split('.')
-        return get_config()[key]
+    data = _config
-    return default
+
+    last = key.pop()
+
+    for item in key:
+        if (not item in data) or (not type(data[item]) == dict):
+            return default
+        data = data[item]
+
+    if not last in data:
+        return default
+
+    return data[last]
 
 def set(key, value = None, savefile = False):
     '''
@@ -38,16 +49,40 @@ def set(key, value = None, savefile = False):
     '''
 
     global _config
+
+    key = str(key).split('.')
+    data = _config
+
+    last = key.pop()
+
+    for item in key:
+        if (not item in data) or (not type(data[item]) == dict):
+            data[item] = dict()
+        data = data[item]
+
     if value is None:
-        del _config[key]
+        del data[last]
     else:
-        _config[key] = value
+        data[last] = value
 
     if savefile:
         save()
 
 def is_set(key):
-    return key in get_config() and not get_config()[key] is None
+    key = str(key).split('.')
+    data = _config
+
+    last = key.pop()
+
+    for item in key:
+        if (not item in data) or (not type(data[item]) == dict):
+            return False
+        data = data[item]
+
+    if not last in data:
+        return False
+
+    return True
 
 def check():
     '''
@@ -71,7 +106,7 @@ def save():
     check()
     try:
         with open(get_config_file(), 'w', encoding="utf8") as configfile:
-            json.dump(get_config(), configfile, indent=2, sort_keys=True)
+            json.dump(get_config(), configfile, indent=2)
     except:
         logger.warn('Failed to write to configuration file.')
 

onionr/core.py

@@ -110,7 +110,7 @@ class Core:
         '''
             Add an address to the address database (only tor currently)
         '''
-        if address == config.get('i2p', {'ownAddr' : None})['ownAddr']:
+        if address == config.get('i2p.ownAddr', None):
 
             return False
         if self._utils.validateID(address):

onionr/onionr.py

@@ -67,22 +67,22 @@ class Onionr:
             config.set_config(json.loads(open('static-data/default_config.json').read())) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
         else:
             # the default config file doesn't exist, try hardcoded config
-            config.set_config({'devmode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}})
+            config.set_config({'dev_mode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}})
         if not data_exists:
             config.save()
         config.reload() # this will read the configuration file into memory
 
         settings = 0b000
-        if config.get('log', {'console': {'color': True}})['console']['color']:
+        if config.get('log.console.color', True):
             settings = settings | logger.USE_ANSI
-        if config.get('log', {'console': {'output': True}})['console']['output']:
+        if config.get('log.console.output', True):
             settings = settings | logger.OUTPUT_TO_CONSOLE
-        if config.get('log', {'file': {'output': True}})['file']['output']:
+        if config.get('log.file.output', True):
             settings = settings | logger.OUTPUT_TO_FILE
-            logger.set_file(config.get('log', {'file': {'path': 'data/output.log'}})['file']['path'])
+            logger.set_file(config.get('log.file.path', '/tmp/onionr.log'))
         logger.set_settings(settings)
 
-        if str(config.get('devmode', True)).lower() == 'true':
+        if str(config.get('general.dev_mode', True)).lower() == 'true':
             self._developmentMode = True
             logger.set_level(logger.LEVEL_DEBUG)
         else:
@@ -149,7 +149,7 @@ class Onionr:
                     randomPort = random.randint(1024, 65535)
                     if self.onionrUtils.checkPort(randomPort):
                         break
-            config.set('client', {'participate': 'true', 'client_hmac': base64.b16encode(os.urandom(32)).decode('utf-8'), 'port': randomPort, 'api_version': API_VERSION}, True)
+            config.set('client', {'participate': 'true', 'hmac': base64.b16encode(os.urandom(32)).decode('utf-8'), 'port': randomPort, 'api_version': API_VERSION}, True)
 
         self.cmds = {
             '': self.showHelpSuggestion,
@@ -262,7 +262,7 @@ class Onionr:
         self.onionrCore.daemonQueueAdd('connectedPeers')
 
     def getWebPassword(self):
-        return config.get('client')['client_hmac']
+        return config.get('client.hmac')
 
     def getHelp(self):
         return self.cmdhelp
@@ -534,12 +534,12 @@ class Onionr:
 
         logger.info('Do ' + logger.colors.bold + sys.argv[0] + ' --help' + logger.colors.reset + logger.colors.fg.green + ' for Onionr help.')
 
-    def start(self, input = False):
+    def start(self, input = False, override = False):
         '''
             Starts the Onionr daemon
         '''
 
-        if os.path.exists('.onionr-lock'):
+        if os.path.exists('.onionr-lock') and not override:
             logger.fatal('Cannot start. Daemon is already running, or it did not exit cleanly.\n(if you are sure that there is not a daemon running, delete .onionr-lock & try again).')
         else:
             if not self.debug and not self._developmentMode:
@@ -560,7 +560,7 @@ class Onionr:
         if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
             if self._developmentMode:
                 logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)', timestamp = False)
-            net = NetController(config.get('client')['port'])
+            net = NetController(config.get('client.port', 59496))
             logger.info('Tor is starting...')
             if not net.startTor():
                 sys.exit(1)
@@ -568,7 +568,7 @@ class Onionr:
             logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey)
             time.sleep(1)
             try:
-                if config.get('newCommunicator'):
+                if config.get('general.newCommunicator', False):
                     communicatorDaemon = './communicator2.py'
                     logger.info('Using new communicator')
             except NameError:
@@ -589,7 +589,7 @@ class Onionr:
         logger.warn('Killing the running daemon...', timestamp = False)
         try:
             events.event('daemon_stop', onionr = self)
-            net = NetController(config.get('client')['port'])
+            net = NetController(config.get('client.port', 59496))
             try:
                 self.onionrUtils.localCommand('shutdown')
             except requests.exceptions.ConnectionError:
@@ -628,7 +628,7 @@ class Onionr:
                 # count stats
                 'div2' : True,
                 'Known Peers Count' : str(len(self.onionrCore.listPeers()) - 1),
-                'Enabled Plugins Count' : str(len(config.get('plugins')['enabled'])) + ' / ' + str(len(os.listdir('data/plugins/'))),
+                'Enabled Plugins Count' : str(len(config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir('data/plugins/'))),
                 'Known Blocks Count' : str(totalBlocks),
                 'Percent Blocks Signed' : str(round(100 * signedBlocks / max(totalBlocks, 1), 2)) + '%'
             }

onionr/onionrblockapi.py

@@ -18,10 +18,13 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 
-import core as onionrcore, logger
+import core as onionrcore, logger, config
-import json, os, datetime, base64
+import json, os, sys, datetime, base64
 
 class Block:
+    blockCacheOrder = list() # NEVER write your own code that writes to this!
+    blockCache = dict() # should never be accessed directly, look at Block.getCache()
+
     def __init__(self, hash = None, core = None):
         '''
             Initializes Onionr
@@ -43,6 +46,9 @@ class Block:
             self.hash = None
             self.core = None
         else:
+            if type(hash) == bytes:
+                hash = hash.decode()
+
             self.btype = ''
             self.bcontent = ''
             self.hash = hash
@@ -65,7 +71,8 @@ class Block:
         if self.getCore() is None:
             self.core = onionrcore.Core()
         if not self.getHash() is None:
-            self.update()
+            if not self.update():
+                logger.debug('Failed to open block %s.' % self.getHash())
 
     # logic
 
@@ -93,16 +100,23 @@ class Block:
             if blockdata is None:
                 filelocation = file
 
+                readfile = True
+
                 if filelocation is None:
                     if self.getHash() is None:
                         return False
-                    try:
+                    elif self.getHash() in Block.getCache():
-                        filelocation = 'data/blocks/%s.dat' % self.getHash().decode()
+                        # get the block from cache, if it's in it
-                    except AttributeError:
+                        blockdata = Block.getCache(self.getHash())
+                        readfile = False
+
+                    # read from file if it's still None
+                    if blockdata is None:
                         filelocation = 'data/blocks/%s.dat' % self.getHash()
 
-                with open(filelocation, 'rb') as f:
+                if readfile:
-                    blockdata = f.read().decode('utf-8')
+                    with open(filelocation, 'rb') as f:
+                        blockdata = f.read().decode()
 
                 self.blockFile = filelocation
             else:
@@ -126,6 +140,10 @@ class Block:
                 self.date = datetime.datetime.fromtimestamp(self.getDate())
 
             self.valid = True
+
+            if len(self.getRaw()) <= config.get('allocations.blockCache', 500000):
+                self.cache()
+
             return True
         except Exception as e:
             logger.error('Failed to update block data.', error = e, timestamp = False)
@@ -641,11 +659,54 @@ class Block:
             - (bool): whether or not the block file exists
         '''
 
+        # no input data? scrap it.
         if hash is None:
             return False
-        elif type(hash) == Block:
+
+        if type(hash) == Block:
             blockfile = hash.getBlockFile()
         else:
             blockfile = 'data/blocks/%s.dat' % hash
 
         return os.path.exists(blockfile) and os.path.isfile(blockfile)
+
+    def getCache(hash = None):
+        # give a list of the hashes of the cached blocks
+        if hash is None:
+            return list(Block.blockCache.keys())
+
+        # if they inputted self or a Block, convert to hash
+        if type(hash) == Block:
+            hash = hash.getHash()
+
+        # just to make sure someone didn't put in a bool or something lol
+        hash = str(hash)
+
+        # if it exists, return its content
+        if hash in Block.getCache():
+            return Block.blockCache[hash]
+
+        return None
+
+    def cache(block, override = False):
+        # why even bother if they're giving bad data?
+        if not type(block) == Block:
+            return False
+
+        # only cache if written to file
+        if block.getHash() is None:
+            return False
+
+        # if it's already cached, what are we here for?
+        if block.getHash() in Block.getCache() and not override:
+            return False
+
+        # dump old cached blocks if the size exeeds the maximum
+        if sys.getsizeof(Block.blockCacheOrder) >= config.get('allocations.blockCacheTotal', 50000000): # 50MB default cache size
+            del Block.blockCache[blockCacheOrder.pop(0)]
+
+        # cache block content
+        Block.blockCache[block.getHash()] = block.getRaw()
+        Block.blockCacheOrder.append(block.getHash())
+
+        return True

onionr/onionrexceptions.py

@@ -18,7 +18,16 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 
+# general exceptions
+class NotFound(Exception):
+    pass
+class Unknown(Exception):
+    pass
+class Invalid(Exception):
+    pass
+
+# network level exceptions
 class MissingPort(Exception):
     pass
 class InvalidAddress(Exception):
-    pass
+    pass

onionr/onionrplugins.py

@@ -64,9 +64,7 @@ def enable(name, onionr = None, start_event = True):
         enabled_plugins = get_enabled_plugins()
         if not name in enabled_plugins:
             enabled_plugins.append(name)
-            config_plugins = config.get('plugins')
+            config.set('plugins.enabled', enabled_plugins, True)
-            config_plugins['enabled'] = enabled_plugins
-            config.set('plugins', config_plugins, True)
 
             events.call(get_plugin(name), 'enable', onionr)
 
@@ -93,9 +91,7 @@ def disable(name, onionr = None, stop_event = True):
     if is_enabled(name):
         enabled_plugins = get_enabled_plugins()
         enabled_plugins.remove(name)
-        config_plugins = config.get('plugins')
+        config.set('plugins.enabled', enabled_plugins, True)
-        config_plugins['enabled'] = enabled_plugins
-        config.set('plugins', config_plugins, True)
 
     if exists(name):
         events.call(get_plugin(name), 'disable', onionr)
@@ -187,7 +183,7 @@ def get_enabled_plugins():
 
     config.reload()
 
-    return config.get('plugins')['enabled']
+    return config.get('plugins.enabled', list())
 
 def is_enabled(name):
     '''

onionr/onionrutils.py

@@ -177,7 +177,7 @@ class OnionrUtils:
         self.getTimeBypassToken()
         # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
         try:
-            retData = requests.get('http://' + open('data/host.txt', 'r').read() + ':' + str(config.get('client')['port']) + '/client/?action=' + command + '&token=' + str(config.get('client')['client_hmac']) + '&timingToken=' + self.timingToken).text
+            retData = requests.get('http://%s:%s/client/?action=%s&token=%s&timingToken=' % (open('data/host.txt', 'r').read(), config.get('client.port', 59496), command, config.get('client.hmac'), self.timingToken)).text
         except Exception as error:
             if not silent:
                 logger.error('Failed to make local request (command: %s).' % command, error=error)

onionr/static-data/default_config.json

@@ -1,8 +1,18 @@
 {
-    "devmode": true,
+    "general" : {
-    "dc_response": true,
+        "dev_mode": true,
+        "display_header" : true,
+
+        "newCommunicator": false,
+
+        "dc_response": true,
+        "dc_execcallbacks" : true
+    },
+
+    "client" : {
+
+    },
 
-    "display_header" : true,
     "log": {
         "file": {
             "output": true,
@@ -15,14 +25,20 @@
         }
     },
 
+    "tor" : {
+
+    },
+
     "i2p":{
-	"host": false,
+    	"host": false,
-    "connect": true,
+        "connect": true,
-    "ownAddr": ""
+        "ownAddr": ""
      },
+
     "allocations":{
         "disk": 1000000000,
-        "netTotal": 1000000000
+        "netTotal": 1000000000,
-    },
+        "blockCache" : 5000000,
-    "newCommunicator": false
+        "blockCacheTotal" : 50000000
+    }
 }