Merge branch 'cleanup2' into dbstorage

.gitlab-ci.yml

@@ -1,6 +0,0 @@
-test:
-  script:
-  - apt-get update -qy
-  - apt-get install -y python3-dev python3-pip tor
-  - pip3 install -r requirements.txt
-  - make test

.travis.yml

@@ -1,8 +0,0 @@
-language: python
-python:
-        - "3.6.4"
-# install dependencies
-install:
-        - sudo apt install tor
-        - pip install -r requirements.txt
-script: make test

onionr-daemon-linux

@@ -0,0 +1,2 @@
+#!/usr/bin/sh
+nohup ./run-linux start & disown

onionr/api.py

@@ -17,22 +17,344 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import flask
+import flask, cgi
 from flask import request, Response, abort, send_from_directory
-from multiprocessing import Process
 from gevent.pywsgi import WSGIServer
 import sys, random, threading, hmac, hashlib, base64, time, math, os, json
 import core
 from onionrblockapi import Block
 import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionr
 
+def guessMime(path):
+    '''
+        Guesses the mime type of a file from the input filename
+    '''
+    mimetypes = {
+        'html' : 'text/html',
+        'js' : 'application/javascript',
+        'css' : 'text/css',
+        'png' : 'image/png',
+        'jpg' : 'image/jpeg'
+    }
+
+    for mimetype in mimetypes:
+        if path.endswith('.%s' % mimetype):
+            return mimetypes[mimetype]
+
+    return 'text/plain'
+
+def setBindIP(filePath):
+    '''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)'''
+    hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
+    data = '.'.join(hostOctets)
+
+    with open(filePath, 'w') as bindFile:
+        bindFile.write(data)
+    return data
+
+class PublicAPI:
+    '''
+        The new client api server, isolated from the public api
+    '''
+    def __init__(self, clientAPI):
+        assert isinstance(clientAPI, API)
+        app = flask.Flask('PublicAPI')
+        self.i2pEnabled = config.get('i2p.host', False)
+        self.hideBlocks = [] # Blocks to be denied sharing
+        self.host = setBindIP(clientAPI._core.publicApiHostFile)
+        self.torAdder = clientAPI._core.hsAddress
+        self.i2pAdder = clientAPI._core.i2pAddress
+        self.bindPort = config.get('client.public.port')
+        logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
+
+        @app.before_request
+        def validateRequest():
+            '''Validate request has the correct hostname'''
+            if type(self.torAdder) is None and type(self.i2pAdder) is None:
+                # abort if our hs addresses are not known
+                abort(403)
+            if request.host not in (self.i2pAdder, self.torAdder):
+                abort(403)
+
+        @app.after_request
+        def sendHeaders(resp):
+            '''Send api, access control headers'''
+            resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch.
+            resp.headers["Content-Security-Policy"] =  "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
+            resp.headers['X-Frame-Options'] = 'deny'
+            resp.headers['X-Content-Type-Options'] = "nosniff"
+            resp.headers['X-API'] = onionr.API_VERSION
+            return resp
+
+        @app.route('/')
+        def banner():
+            try:
+                with open('static-data/index.html', 'r') as html:
+                    resp = Response(html.read(), mimetype='text/html')
+            except FileNotFoundError:
+                resp = Response("")
+            return resp
+
+        @app.route('/getblocklist')
+        def getBlockList():
+            bList = clientAPI._core.getBlockList()
+            for b in self.hideBlocks:
+                if b in bList:
+                    bList.remove(b)
+            return Response('\n'.join(bList))
+
+        @app.route('/getdata/<name>')
+        def getBlockData(name):
+            resp = ''
+            data = name
+            if clientAPI._utils.validateHash(data):
+                if data not in self.hideBlocks:
+                    if os.path.exists(clientAPI._core.dataDir + 'blocks/' + data + '.dat'):
+                        block = Block(hash=data.encode(), core=clientAPI._core)
+                        resp = base64.b64encode(block.getRaw().encode()).decode()
+            if len(resp) == 0:
+                abort(404)
+                resp = ""
+            return Response(resp)
+
+        @app.route('/www/<path:path>')
+        def wwwPublic(path):
+            if not config.get("www.public.run", True):
+                abort(403)
+            return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
+
+        @app.route('/ping')
+        def ping():
+            return Response("pong!")
+
+        @app.route('/getdbhash')
+        def getDBHash():
+            return Response(clientAPI._utils.getBlockDBHash())
+
+        @app.route('/pex')
+        def peerExchange():
+            response = ','.join(clientAPI._core.listAdders())
+            if len(response) == 0:
+                response = 'none'
+            return Response(response)
+        
+        @app.route('/announce', methods=['post'])
+        def acceptAnnounce():
+            resp = 'failure'
+            powHash = ''
+            randomData = ''
+            newNode = ''
+            ourAdder = clientAPI._core.hsAddress.encode()
+            try:
+                newNode = request.form['node'].encode()
+            except KeyError:
+                logger.warn('No block specified for upload')
+                pass
+            else:
+                try:
+                    randomData = request.form['random']
+                    randomData = base64.b64decode(randomData)
+                except KeyError:
+                    logger.warn('No random data specified for upload')
+                else:
+                    nodes = newNode + clientAPI._core.hsAddress.encode()
+                    nodes = clientAPI._core._crypto.blake2bHash(nodes)
+                    powHash = clientAPI._core._crypto.blake2bHash(randomData + nodes)
+                    try:
+                        powHash = powHash.decode()
+                    except AttributeError:
+                        pass
+                    if powHash.startswith('0000'):
+                        try:
+                            newNode = newNode.decode()
+                        except AttributeError:
+                            pass
+                        if clientAPI._core.addAddress(newNode):
+                            resp = 'Success'
+                    else:
+                        logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)
+            resp = Response(resp)
+            return resp
+
+        @app.route('/upload', methods=['post'])
+        def upload():
+            resp = 'failure'
+            try:
+                data = request.form['block']
+            except KeyError:
+                logger.warn('No block specified for upload')
+                pass
+            else:
+                if sys.getsizeof(data) < 100000000:
+                    try:
+                        if blockimporter.importBlockFromData(data, clientAPI._core):
+                            resp = 'success'
+                        else:
+                            logger.warn('Error encountered importing uploaded block')
+                    except onionrexceptions.BlacklistedBlock:
+                        logger.debug('uploaded block is blacklisted')
+                        pass
+            if resp == 'failure':
+                abort(400)
+            resp = Response(resp)
+            return resp
+
+        clientAPI.setPublicAPIInstance(self)
+        while self.torAdder == '':
+            clientAPI._core.refreshFirstStartVars()
+            self.torAdder = clientAPI._core.hsAddress
+            time.sleep(1)
+        self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None)
+        self.httpServer.serve_forever()
+
 class API:
     '''
-        Main HTTP API (Flask)
+        Client HTTP api
     '''
 
     callbacks = {'public' : {}, 'private' : {}}
 
+    def __init__(self, onionrInst, debug, API_VERSION):
+        '''
+            Initialize the api server, preping variables for later use
+
+            This initilization defines all of the API entry points and handlers for the endpoints and errors
+            This also saves the used host (random localhost IP address) to the data folder in host.txt
+        '''
+        # assert isinstance(onionrInst, onionr.Onionr)
+        # configure logger and stuff
+        onionr.Onionr.setupConfig('data/', self = self)
+
+        self.debug = debug
+        self._privateDelayTime = 3
+        self._core = core.Core()
+        self._crypto = onionrcrypto.OnionrCrypto(self._core)
+        self._utils = onionrutils.OnionrUtils(self._core)
+        app = flask.Flask(__name__)
+        bindPort = int(config.get('client.client.port', 59496))
+        self.bindPort = bindPort
+
+        self.whitelistEndpoints = ('site', 'www', 'onionrhome', 'board', 'boardContent', 'sharedContent')
+
+        self.clientToken = config.get('client.webpassword')
+        self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
+
+        self.publicAPI = None # gets set when the thread calls our setter... bad hack but kinda necessary with flask
+        #threading.Thread(target=PublicAPI, args=(self,)).start()
+        self.host = setBindIP(self._core.privateApiHostFile)
+        logger.info('Running api on %s:%s' % (self.host, self.bindPort))
+        self.httpServer = ''
+        onionrInst.setClientAPIInst(self)
+
+        @app.before_request
+        def validateRequest():
+            '''Validate request has set password and is the correct hostname'''
+            if request.host != '%s:%s' % (self.host, self.bindPort):
+                abort(403)
+            if request.endpoint in self.whitelistEndpoints:
+                return
+            try:
+                if not hmac.compare_digest(request.headers['token'], self.clientToken):
+                    abort(403)
+            except KeyError:
+                abort(403)
+
+        @app.after_request
+        def afterReq(resp):
+            #resp.headers["Content-Security-Policy"] =  "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
+            resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self'; img-src 'self'; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'self'"
+            resp.headers['X-Frame-Options'] = 'deny'
+            resp.headers['X-Content-Type-Options'] = "nosniff"
+            resp.headers['X-API'] = onionr.API_VERSION
+            resp.headers['Server'] = ''
+            resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch.
+            return resp
+
+        @app.route('/board/', endpoint='board')
+        def loadBoard():
+            return send_from_directory('static-data/www/board/', "index.html")
+
+        @app.route('/board/<path:path>', endpoint='boardContent')
+        def boardContent(path):
+            return send_from_directory('static-data/www/board/', path)
+        @app.route('/shared/<path:path>', endpoint='sharedContent')
+        def sharedContent(path):
+            return send_from_directory('static-data/www/shared/', path)
+
+        @app.route('/www/<path:path>', endpoint='www')
+        def wwwPublic(path):
+            if not config.get("www.private.run", True):
+                abort(403)
+            return send_from_directory(config.get('www.private.path', 'static-data/www/private/'), path)
+
+        @app.route('/ping')
+        def ping():
+            return Response("pong!")
+
+        @app.route('/', endpoint='onionrhome')
+        def hello():
+            return Response("Welcome to Onionr")
+        
+        @app.route('/getblocksbytype/<name>')
+        def getBlocksByType(name):
+            blocks = self._core.getBlocksByType(name)
+            return Response(','.join(blocks))
+        
+        @app.route('/gethtmlsafeblockdata/<name>')
+        def getData(name):
+            resp = ''
+            if self._core._utils.validateHash(name):
+                try:
+                    resp =  cgi.escape(Block(name).bcontent, quote=True)
+                except TypeError:
+                    pass
+            else:
+                abort(404)
+            return Response(resp)
+
+        @app.route('/site/<name>', endpoint='site')
+        def site(name):
+            bHash = name
+            resp = 'Not Found'
+            if self._core._utils.validateHash(bHash):
+                try:
+                    resp = Block(bHash).bcontent
+                except TypeError:
+                    pass
+                try:
+                    resp = base64.b64decode(resp)
+                except:
+                    pass
+            if resp == 'Not Found':
+                abourt(404)
+            return Response(resp)
+
+        @app.route('/waitforshare/<name>', methods=['post'])
+        def waitforshare():
+            assert name.isalnum()
+            if name in self.publicAPI.hideBlocks:
+                self.publicAPI.hideBlocks.remove(name)
+                return Response("removed")
+            else:
+                self.publicAPI.hideBlocks.append(name)
+                return Response("added")
+
+        @app.route('/shutdown')
+        def shutdown():
+            try:
+                self.publicAPI.httpServer.stop()
+                self.httpServer.stop()
+            except AttributeError:
+                pass
+            return Response("bye")
+
+        self.httpServer = WSGIServer((self.host, bindPort), app, log=None)
+        self.httpServer.serve_forever()
+
+    def setPublicAPIInstance(self, inst):
+        assert isinstance(inst, PublicAPI)
+        self.publicAPI = inst
+
     def validateToken(self, token):
         '''
             Validate that the client token matches the given token
@@ -47,557 +369,3 @@ class API:
                 return True
         except TypeError:
             return False
-
-    def guessMime(path):
-        '''
-            Guesses the mime type from the input filename
-        '''
-
-        mimetypes = {
-            'html' : 'text/html',
-            'js' : 'application/javascript',
-            'css' : 'text/css',
-            'png' : 'image/png',
-            'jpg' : 'image/jpeg'
-        }
-
-        for mimetype in mimetypes:
-            if path.endswith('.%s' % mimetype):
-                return mimetypes[mimetype]
-
-        return 'text/plain'
-
-    def __init__(self, debug, API_VERSION):
-        '''
-            Initialize the api server, preping variables for later use
-
-            This initilization defines all of the API entry points and handlers for the endpoints and errors
-            This also saves the used host (random localhost IP address) to the data folder in host.txt
-        '''
-
-        # configure logger and stuff
-        onionr.Onionr.setupConfig('data/', self = self)
-
-        self.debug = debug
-        self._privateDelayTime = 3
-        self._core = core.Core()
-        self._crypto = onionrcrypto.OnionrCrypto(self._core)
-        self._utils = onionrutils.OnionrUtils(self._core)
-        app = flask.Flask(__name__)
-        bindPort = int(config.get('client.port', 59496))
-        self.bindPort = bindPort
-        self.clientToken = config.get('client.webpassword')
-        self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
-
-        self.i2pEnabled = config.get('i2p.host', False)
-
-        self.hideBlocks = [] # Blocks to be denied sharing
-
-        with open(self._core.dataDir + 'time-bypass.txt', 'w') as bypass:
-            bypass.write(self.timeBypassToken)
-
-        if not debug and not self._developmentMode:
-            hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
-            self.host = '.'.join(hostOctets)
-        else:
-            self.host = '127.0.0.1'
-
-        with open(self._core.dataDir + 'host.txt', 'w') as file:
-            file.write(self.host)
-
-        @app.before_request
-        def beforeReq():
-            '''
-                Simply define the request as not having yet failed, before every request.
-            '''
-            self.requestFailed = False
-            return
-
-        @app.after_request
-        def afterReq(resp):
-            if not self.requestFailed:
-                resp.headers['Access-Control-Allow-Origin'] = '*'
-            #else:
-            #    resp.headers['server'] = 'Onionr'
-            resp.headers["Content-Security-Policy"] =  "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
-            resp.headers['X-Frame-Options'] = 'deny'
-            resp.headers['X-Content-Type-Options'] = "nosniff"
-            resp.headers['X-API'] = API_VERSION
-            resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch.
-            return resp
-
-        @app.route('/site/<path:block>')
-        def site(block):
-            self.validateHost('private')
-            bHash = block
-            resp = 'Not Found'
-            if self._core._utils.validateHash(bHash):
-                resp = Block(bHash).bcontent
-                try:
-                    resp = base64.b64decode(resp)
-                except:
-                    pass
-            return Response(resp)
-
-        @app.route('/www/private/<path:path>')
-        def www_private(path):
-            startTime = math.floor(time.time())
-
-            if request.args.get('timingToken') is None:
-                timingToken = ''
-            else:
-                timingToken = request.args.get('timingToken')
-
-            if not config.get("www.private.run", True):
-                abort(403)
-
-            self.validateHost('private')
-
-            endTime = math.floor(time.time())
-            elapsed = endTime - startTime
-
-            if not hmac.compare_digest(timingToken, self.timeBypassToken):
-                if (elapsed < self._privateDelayTime) and config.get('www.private.timing_protection', True):
-                    time.sleep(self._privateDelayTime - elapsed)
-
-            return send_from_directory(config.get('www.private.path', 'static-data/www/private/'), path)
-
-        @app.route('/www/public/<path:path>')
-        def www_public(path):
-            if not config.get("www.public.run", True):
-                abort(403)
-
-            self.validateHost('public')
-
-            return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
-
-        @app.route('/ui/<path:path>')
-        def ui_private(path):
-            startTime = math.floor(time.time())
-
-            '''
-            if request.args.get('timingToken') is None:
-                timingToken = ''
-            else:
-                timingToken = request.args.get('timingToken')
-            '''
-
-            if not config.get("www.ui.run", True):
-                abort(403)
-
-            if config.get("www.ui.private", True):
-                self.validateHost('private')
-            else:
-                self.validateHost('public')
-
-            '''
-            endTime = math.floor(time.time())
-            elapsed = endTime - startTime
-
-            if not hmac.compare_digest(timingToken, self.timeBypassToken):
-                if elapsed < self._privateDelayTime:
-                    time.sleep(self._privateDelayTime - elapsed)
-            '''
-
-            mime = API.guessMime(path)
-
-            logger.debug('Serving %s (mime: %s)' % (path, mime))
-
-            return send_from_directory('static-data/www/ui/dist/', path)
-
-        @app.route('/client/')
-        def private_handler():
-            if request.args.get('timingToken') is None:
-                timingToken = ''
-            else:
-                timingToken = request.args.get('timingToken')
-            data = request.args.get('data')
-            try:
-                data = data
-            except:
-                data = ''
-            startTime = math.floor(time.time())
-
-            action = request.args.get('action')
-            #if not self.debug:
-            token = request.args.get('token')
-
-            if not self.validateToken(token):
-                abort(403)
-
-            events.event('webapi_private', onionr = None, data = {'action' : action, 'data' : data, 'timingToken' : timingToken, 'token' : token})
-
-            self.validateHost('private')
-            if action == 'hello':
-                resp = Response('Hello, World! ' + request.host)
-            elif action == 'getIP':
-                resp = Response(self.host)
-            elif action == 'waitForShare':
-                if self._core._utils.validateHash(data):
-                    if data not in self.hideBlocks:
-                        self.hideBlocks.append(data)
-                    else:
-                        self.hideBlocks.remove(data)
-                    resp = "success"
-                else:
-                    resp = "failed to validate hash"
-            elif action == 'shutdown':
-                # request.environ.get('werkzeug.server.shutdown')()
-                self.http_server.stop()
-                resp = Response('Goodbye')
-            elif action == 'ping':
-                resp = Response('pong')
-            elif action == 'info':
-                resp = Response(json.dumps({'pubkey' : self._core._crypto.pubKey, 'host' : self._core.hsAddress}), mimetype='text/plain')
-            elif action == "insertBlock":
-                response = {'success' : False, 'reason' : 'An unknown error occurred'}
-
-                if not ((data is None) or (len(str(data).strip()) == 0)):
-                    try:
-                        decoded = json.loads(data)
-
-                        block = Block()
-
-                        sign = False
-
-                        for key in decoded:
-                            val = decoded[key]
-
-                            key = key.lower()
-
-                            if key == 'type':
-                                block.setType(val)
-                            elif key in ['body', 'content']:
-                                block.setContent(val)
-                            elif key == 'parent':
-                                block.setParent(val)
-                            elif key == 'sign':
-                                sign = (str(val).lower() == 'true')
-
-                        hash = block.save(sign = sign)
-
-                        if not hash is False:
-                            response['success'] = True
-                            response['hash'] = hash
-                            response['reason'] = 'Successfully wrote block to file'
-                        else:
-                            response['reason'] = 'Failed to save the block'
-                    except Exception as e:
-                        logger.warn('insertBlock api request failed', error = e)
-                        logger.debug('Here\'s the request: %s' % data)
-                else:
-                    response = {'success' : False, 'reason' : 'Missing `data` parameter.', 'blocks' : {}}
-
-                resp = Response(json.dumps(response))
-            elif action == 'searchBlocks':
-                response = {'success' : False, 'reason' : 'An unknown error occurred', 'blocks' : {}}
-
-                if not ((data is None) or (len(str(data).strip()) == 0)):
-                    try:
-                        decoded = json.loads(data)
-
-                        type = None
-                        signer = None
-                        signed = None
-                        parent = None
-                        reverse = False
-                        limit = None
-
-                        for key in decoded:
-                            val = decoded[key]
-
-                            key = key.lower()
-
-                            if key == 'type':
-                                type = str(val)
-                            elif key == 'signer':
-                                if isinstance(val, list):
-                                    signer = val
-                                else:
-                                    signer = str(val)
-                            elif key == 'signed':
-                                signed = (str(val).lower() == 'true')
-                            elif key == 'parent':
-                                parent = str(val)
-                            elif key == 'reverse':
-                                reverse = (str(val).lower() == 'true')
-                            elif key == 'limit':
-                                limit = 10000
-
-                                if val is None:
-                                    val = limit
-
-                                limit = min(limit, int(val))
-
-                        blockObjects = Block.getBlocks(type = type, signer = signer, signed = signed, parent = parent, reverse = reverse, limit = limit)
-
-                        logger.debug('%s results for query %s' % (len(blockObjects), decoded))
-
-                        blocks = list()
-
-                        for block in blockObjects:
-                            blocks.append({
-                                'hash' : block.getHash(),
-                                'type' : block.getType(),
-                                'content' : block.getContent(),
-                                'signature' : block.getSignature(),
-                                'signedData' : block.getSignedData(),
-                                'signed' : block.isSigned(),
-                                'valid' : block.isValid(),
-                                'date' : (int(block.getDate().strftime("%s")) if not block.getDate() is None else None),
-                                'parent' : (block.getParent().getHash() if not block.getParent() is None else None),
-                                'metadata' : block.getMetadata(),
-                                'header' : block.getHeader()
-                            })
-
-                        response['success'] = True
-                        response['blocks'] = blocks
-                        response['reason'] = 'Success'
-                    except Exception as e:
-                        logger.warn('searchBlock api request failed', error = e)
-                        logger.debug('Here\'s the request: %s' % data)
-                else:
-                    response = {'success' : False, 'reason' : 'Missing `data` parameter.', 'blocks' : {}}
-
-                resp = Response(json.dumps(response), mimetype='text/plain')
-
-            elif action in API.callbacks['private']:
-                resp = Response(str(getCallback(action, scope = 'private')(request)), mimetype='text/plain')
-            else:
-                resp = Response('invalid command')
-            endTime = math.floor(time.time())
-            elapsed = endTime - startTime
-
-            # if bypass token not used, delay response to prevent timing attacks
-            if not hmac.compare_digest(timingToken, self.timeBypassToken):
-                if elapsed < self._privateDelayTime:
-                    time.sleep(self._privateDelayTime - elapsed)
-
-            return resp
-
-        @app.route('/')
-        def banner():
-            self.validateHost('public')
-            try:
-                with open('static-data/index.html', 'r') as html:
-                    resp = Response(html.read(), mimetype='text/html')
-            except FileNotFoundError:
-                resp = Response("")
-            return resp
-
-        @app.route('/public/upload/', methods=['POST'])
-        def blockUpload():
-            self.validateHost('public')
-            resp = 'failure'
-            try:
-                data = request.form['block']
-            except KeyError:
-                logger.warn('No block specified for upload')
-                pass
-            else:
-                if sys.getsizeof(data) < 100000000:
-                    try:
-                        if blockimporter.importBlockFromData(data, self._core):
-                            resp = 'success'
-                        else:
-                            logger.warn('Error encountered importing uploaded block')
-                    except onionrexceptions.BlacklistedBlock:
-                        logger.debug('uploaded block is blacklisted')
-                        pass
-
-            resp = Response(resp)
-            return resp
-
-        @app.route('/public/announce/', methods=['POST'])
-        def acceptAnnounce():
-            self.validateHost('public')
-            resp = 'failure'
-            powHash = ''
-            randomData = ''
-            newNode = ''
-            ourAdder = self._core.hsAddress.encode()
-            try:
-                newNode = request.form['node'].encode()
-            except KeyError:
-                logger.warn('No block specified for upload')
-                pass
-            else:
-                try:
-                    randomData = request.form['random']
-                    randomData = base64.b64decode(randomData)
-                except KeyError:
-                    logger.warn('No random data specified for upload')
-                else:
-                    nodes = newNode + self._core.hsAddress.encode()
-                    nodes = self._core._crypto.blake2bHash(nodes)
-                    powHash = self._core._crypto.blake2bHash(randomData + nodes)
-                    try:
-                        powHash = powHash.decode()
-                    except AttributeError:
-                        pass
-                    if powHash.startswith('0000'):
-                        try:
-                            newNode = newNode.decode()
-                        except AttributeError:
-                            pass
-                        if self._core.addAddress(newNode):
-                            resp = 'Success'
-                    else:
-                        logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)
-            resp = Response(resp)
-            return resp
-
-        @app.route('/public/')
-        def public_handler():
-            # Public means it is publicly network accessible
-            self.validateHost('public')
-            if config.get('general.security_level') != 0:
-                abort(403)
-            action = request.args.get('action')
-            requestingPeer = request.args.get('myID')
-            data = request.args.get('data')
-            try:
-                data = data
-            except:
-                data = ''
-
-            events.event('webapi_public', onionr = None, data = {'action' : action, 'data' : data, 'requestingPeer' : requestingPeer, 'request' : request})
-
-            if action == 'firstConnect':
-                pass
-            elif action == 'ping':
-                resp = Response("pong!")
-            elif action == 'getSymmetric':
-                resp = Response(self._crypto.generateSymmetric())
-            elif action == 'getDBHash':
-                resp = Response(self._utils.getBlockDBHash())
-            elif action == 'getBlockHashes':
-                bList = self._core.getBlockList()
-                for b in self.hideBlocks:
-                    if b in bList:
-                        bList.remove(b)
-                resp = Response('\n'.join(bList))
-            # setData should be something the communicator initiates, not this api
-            elif action == 'getData':
-                resp = ''
-                if self._utils.validateHash(data):
-                    if data not in self.hideBlocks:
-                        if os.path.exists(self._core.dataDir + 'blocks/' + data + '.dat'):
-                            block = Block(hash=data.encode(), core=self._core)
-                            resp = base64.b64encode(block.getRaw().encode()).decode()
-                if len(resp) == 0:
-                    abort(404)
-                    resp = ""
-                resp = Response(resp)
-            elif action == 'pex':
-                response = ','.join(self._core.listAdders())
-                if len(response) == 0:
-                    response = 'none'
-                resp = Response(response)
-            elif action == 'kex':
-                peers = self._core.listPeers(getPow=True)
-                response = ','.join(peers)
-                resp = Response(response)
-            elif action in API.callbacks['public']:
-                resp = Response(str(getCallback(action, scope = 'public')(request)))
-            else:
-                resp = Response("")
-
-            return resp
-
-        @app.errorhandler(404)
-        def notfound(err):
-            self.requestFailed = True
-            resp = Response("")
-
-            return resp
-
-        @app.errorhandler(403)
-        def authFail(err):
-            self.requestFailed = True
-            resp = Response("403")
-            return resp
-
-        @app.errorhandler(401)
-        def clientError(err):
-            self.requestFailed = True
-            resp = Response("Invalid request")
-
-            return resp
-
-        logger.info('Starting client on ' + self.host + ':' + str(bindPort), timestamp=False)
-
-        try:
-            while len(self._core.hsAddress) == 0:
-                self._core.refreshFirstStartVars()
-                time.sleep(0.5)
-            self.http_server = WSGIServer((self.host, bindPort), app, log=None)
-            self.http_server.serve_forever()
-        except KeyboardInterrupt:
-            pass
-        except Exception as e:
-            logger.error(str(e))
-            logger.fatal('Failed to start client on ' + self.host + ':' + str(bindPort) + ', exiting...')
-
-    def validateHost(self, hostType):
-        '''
-            Validate various features of the request including:
-
-            If private (/client/), is the host header local?
-            If public (/public/), is the host header onion or i2p?
-
-            Was X-Request-With used?
-        '''
-        if self.debug:
-            return
-        # Validate host header, to protect against DNS rebinding attacks
-        host = self.host
-        if hostType == 'private':
-            if not request.host.startswith('127') and not self._utils.checkIsIP(request.host):
-                abort(403)
-        elif hostType == 'public':
-            if not request.host.endswith('onion') and not request.host.endswith('i2p'):
-                abort(403)
-        # Validate x-requested-with, to protect against CSRF/metadata leaks
-
-        if not self.i2pEnabled and request.host.endswith('i2p'):
-            abort(403)
-
-        '''
-        if not self._developmentMode:
-            try:
-                request.headers['X-Requested-With']
-            except:
-                pass
-            # we exit rather than abort to avoid fingerprinting
-            logger.debug('Avoiding fingerprinting, exiting...')
-                #sys.exit(1)
-        '''
-
-    def setCallback(action, callback, scope = 'public'):
-        if not scope in API.callbacks:
-            return False
-
-        API.callbacks[scope][action] = callback
-
-        return True
-
-    def removeCallback(action, scope = 'public'):
-        if (not scope in API.callbacks) or (not action in API.callbacks[scope]):
-            return False
-
-        del API.callbacks[scope][action]
-
-        return True
-
-    def getCallback(action, scope = 'public'):
-        if (not scope in API.callbacks) or (not action in API.callbacks[scope]):
-            return None
-
-        return API.callbacks[scope][action]
-
-    def getCallbacks(scope = None):
-        if (not scope is None) and (scope in API.callbacks):
-            return API.callbacks[scope]
-
-        return API.callbacks

onionr/apimanager.py

@@ -0,0 +1,75 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Handles api data exchange, interfaced by both public and client http api
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import config, apipublic, apiprivate, core, socket, random, threading, time
+config.reload()
+
+PRIVATE_API_VERSION = 0
+PUBLIC_API_VERSION = 1
+
+DEV_MODE = config.get('general.dev_mode')
+
+def getOpenPort():
+    '''Get a random open port'''
+    p = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    p.bind(("127.0.0.1",0))
+    p.listen(1)
+    port = p.getsockname()[1]
+    p.close()
+    return port
+
+def getRandomLocalIP():
+    '''Get a random local ip address'''
+    hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
+    host = '.'.join(hostOctets)
+    return host
+
+class APIManager:
+    def __init__(self, coreInst):
+        assert isinstance(coreInst, core.Core)
+        self.core = coreInst
+        self.utils = coreInst._utils
+        self.crypto = coreInst._crypto
+        
+        # if this gets set to true, both the public and private apis will shutdown
+        self.shutdown = False
+        
+        publicIP = '127.0.0.1'
+        privateIP = '127.0.0.1'
+        if DEV_MODE:
+            # set private and local api servers bind IPs to random localhost (127.x.x.x), make sure not the same
+            privateIP = getRandomLocalIP()
+            while True:
+                publicIP = getRandomLocalIP()
+                if publicIP != privateIP:
+                    break
+        
+        # Make official the IPs and Ports
+        self.publicIP = publicIP
+        self.privateIP = privateIP
+        self.publicPort = config.get('client.port', 59496)
+        self.privatePort = config.get('client.port', 59496)
+
+        # Run the API servers in new threads
+        self.publicAPI = apipublic.APIPublic(self)
+        self.privateAPI = apiprivate.APIPrivate(self)
+        threading.Thread(target=self.publicAPI.run).start()
+        threading.Thread(target=self.privateAPI.run).start()
+        while not self.shutdown:
+            time.sleep(1)

onionr/apiprivate.py

@@ -0,0 +1,32 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Handle incoming commands from the client. Intended for localhost use
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import flask, apimanager
+from flask import request, Response, abort, send_from_directory
+from gevent.pywsgi import WSGIServer
+
+class APIPrivate:
+    def __init__(self, managerInst):
+        assert isinstance(managerInst, apimanager.APIManager)
+        self.app = flask.Flask(__name__) # The flask application, which recieves data from the greenlet wsgiserver
+        self.httpServer = WSGIServer((managerInst.privateIP, managerInst.privatePort), self.app, log=None)
+
+    def run(self):
+        self.httpServer.serve_forever()
+        return

onionr/apipublic.py

@@ -0,0 +1,41 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Handle incoming commands from other Onionr nodes, over HTTP
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import flask, apimanager
+from flask import request, Response, abort, send_from_directory
+from gevent.pywsgi import WSGIServer
+
+
+class APIPublic:
+    def __init__(self, managerInst):
+        assert isinstance(managerInst, apimanager.APIManager)
+        app = flask.Flask(__name__)
+        @app.route('/')
+        def banner():
+            try:
+                with open('static-data/index.html', 'r') as html:
+                    resp = Response(html.read(), mimetype='text/html')
+            except FileNotFoundError:
+                resp = Response("")
+            return resp
+        self.httpServer = WSGIServer((managerInst.publicIP, managerInst.publicPort), app)
+
+    def run(self):
+        self.httpServer.serve_forever()
+        return

onionr/communicator2.py

@@ -22,6 +22,7 @@
 import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid
 import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
 import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs
+import binascii
 from dependencies import secrets
 from defusedxml import minidom
 
@@ -101,6 +102,7 @@ class OnionrCommunicatorDaemon:
         OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True)
         OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1)
         OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1)
+        OnionrCommunicatorTimers(self, self.detectAPICrash, 30, maxThreads=1)
         deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1)
 
         netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)
@@ -178,14 +180,14 @@ class OnionrCommunicatorDaemon:
                     break
                 else:
                     continue
-            newDBHash = self.peerAction(peer, 'getDBHash') # get their db hash
+            newDBHash = self.peerAction(peer, 'getdbhash') # get their db hash
             if newDBHash == False or not self._core._utils.validateHash(newDBHash):
                 continue # if request failed, restart loop (peer is added to offline peers automatically)
             triedPeers.append(peer)
             if newDBHash != self._core.getAddressInfo(peer, 'DBHash'):
                 self._core.setAddressInfo(peer, 'DBHash', newDBHash)
                 try:
-                    newBlocks = self.peerAction(peer, 'getBlockHashes') # get list of new block hashes
+                    newBlocks = self.peerAction(peer, 'getblocklist') # get list of new block hashes
                 except Exception as error:
                     logger.warn('Could not get new blocks from %s.' % peer, error = error)
                     newBlocks = False
@@ -224,13 +226,16 @@ class OnionrCommunicatorDaemon:
             self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
             logger.info("Attempting to download %s..." % blockHash)
             peerUsed = self.pickOnlinePeer()
-            content = self.peerAction(peerUsed, 'getData', data=blockHash) # block content from random peer (includes metadata)
+            content = self.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata)
             if content != False and len(content) > 0:
                 try:
                     content = content.encode()
                 except AttributeError:
                     pass
+                try:
                     content = base64.b64decode(content) # content is base64 encoded in transport
+                except binascii.Error:
+                    pass
                 realHash = self._core._crypto.sha3Hash(content)
                 try:
                     realHash = realHash.decode() # bytes on some versions for some reason
@@ -421,7 +426,7 @@ class OnionrCommunicatorDaemon:
         if len(peer) == 0:
             return False
         #logger.debug('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort))
-        url = 'http://' + peer + '/public/?action=' + action
+        url = 'http://%s/%s' % (peer, action)
         if len(data) > 0:
             url += '&data=' + data
 
@@ -518,7 +523,7 @@ class OnionrCommunicatorDaemon:
                     if peer in triedPeers:
                         continue
                     triedPeers.append(peer)
-                    url = 'http://' + peer + '/public/upload/'
+                    url = 'http://' + peer + '/upload'
                     data = {'block': block.Block(bl).getRaw()}
                     proxyType = ''
                     if peer.endswith('.onion'):
@@ -527,7 +532,7 @@ class OnionrCommunicatorDaemon:
                         proxyType = 'i2p'
                     logger.info("Uploading block to " + peer)
                     if not self._core._utils.doPostRequest(url, data=data, proxyType=proxyType) == False:
-                        self._core._utils.localCommand('waitForShare', data=bl)
+                        self._core._utils.localCommand('waitforshare/' + bl)
                         finishedUploads.append(bl)
                         break
         for x in finishedUploads:
@@ -544,9 +549,9 @@ class OnionrCommunicatorDaemon:
 
     def detectAPICrash(self):
         '''exit if the api server crashes/stops'''
-        if self._core._utils.localCommand('ping', silent=False) != 'pong':
+        if self._core._utils.localCommand('ping', silent=False) not in ('pong', 'pong!'):
             for i in range(5):
-                if self._core._utils.localCommand('ping') == 'pong':
+                if self._core._utils.localCommand('ping') in ('pong', 'pong!'):
                     break # break for loop
                 time.sleep(1)
             else:

onionr/core.py

@@ -50,8 +50,11 @@ class Core:
             self.blockDB = self.dataDir + 'blocks.db'
             self.blockDataDB = self.dataDir + 'block-data.db'
             self.blockDataLocation = self.dataDir + 'blocks/'
+            self.publicApiHostFile = self.dataDir + 'public-host.txt'
+            self.privateApiHostFile = self.dataDir + 'private-host.txt'
             self.addressDB = self.dataDir + 'address.db'
             self.hsAddress = ''
+            self.i2pAddress = config.get('i2p.ownAddr', None)
             self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
             self.bootstrapList = []
             self.requirements = onionrvalues.OnionrValues()
@@ -152,7 +155,7 @@ class Core:
 
         if address == config.get('i2p.ownAddr', None) or address == self.hsAddress:
             return False
-        if type(address) is type(None) or len(address) == 0:
+        if type(address) is None or len(address) == 0:
             return False
         if self._utils.validateID(address):
             conn = sqlite3.connect(self.addressDB, timeout=10)
@@ -258,7 +261,7 @@ class Core:
             return
         conn = sqlite3.connect(self.blockDB, timeout=10)
         c = conn.cursor()
-        currentTime = self._utils.getEpoch()
+        currentTime = self._utils.getEpoch() + self._crypto.secrets.randbelow(301)
         if selfInsert or dataSaved:
             selfInsert = 1
         else:
@@ -678,7 +681,10 @@ class Core:
             Inserts a block into the network
             encryptType must be specified to encrypt a block
         '''
-
+        allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
+        if self._utils.storageCounter.isFull():
+            logger.error(allocationReachedMessage)
+            return False
         retData = False
         # check nonce
         dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data))
@@ -761,7 +767,7 @@ class Core:
         metadata['meta'] = jsonMeta
         metadata['sig'] = signature
         metadata['signer'] = signer
-        metadata['time'] = self._utils.getRoundedEpoch() + self._crypto.secrets.randbelow(301)
+        metadata['time'] = self._utils.getRoundedEpoch()
 
         # ensure expire is integer and of sane length
         if type(expire) is not type(None):
@@ -772,9 +778,14 @@ class Core:
         proof = onionrproofs.POW(metadata, data)
         payload = proof.waitForResult()
         if payload != False:
+            try:
                 retData = self.setData(payload)
+            except onionrexceptions.DiskAllocationReached:
+                logger.error(allocationReachedMessage)
+                retData = False
+            else:
                 # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult
-            self._utils.localCommand('waitForShare', data=retData)
+                self._utils.localCommand('waitforshare/' + retData)
                 self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
                 #self.setBlockType(retData, meta['type'])
                 self._utils.processBlockMetadata(retData)

onionr/netcontroller.py

@@ -18,10 +18,19 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 
-import subprocess, os, random, sys, logger, time, signal, config, base64
+import subprocess, os, random, sys, logger, time, signal, config, base64, socket
 from stem.control import Controller
 from onionrblockapi import Block
 from dependencies import secrets
+
+def getOpenPort():
+    # taken from (but modified) https://stackoverflow.com/a/2838309
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    s.bind(("127.0.0.1",0))
+    s.listen(1)
+    port = s.getsockname()[1]
+    s.close()
+    return port
 class NetController:
     '''
         This class handles hidden service setup on Tor and I2P
@@ -37,7 +46,7 @@ class NetController:
 
         self.torConfigLocation = self.dataDir + 'torrc'
         self.readyState = False
-        self.socksPort = random.randint(1024, 65535)
+        self.socksPort = getOpenPort()
         self.hsPort = hsPort
         self._torInstnace = ''
         self.myID = ''
@@ -68,7 +77,7 @@ class NetController:
         hsVer = '# v2 onions'
         if config.get('tor.v3onions'):
             hsVer = 'HiddenServiceVersion 3'
-            logger.debug('Using v3 onions :)')
+            logger.debug('Using v3 onions')
 
         if os.path.exists(self.torConfigLocation):
             os.remove(self.torConfigLocation)
@@ -78,7 +87,7 @@ class NetController:
         config.set('tor.controlpassword', plaintext, savefile=True)
         config.set('tor.socksport', self.socksPort, savefile=True)
 
-        controlPort = random.randint(1025, 65535)
+        controlPort = getOpenPort()
 
         config.set('tor.controlPort', controlPort, savefile=True)
 

onionr/onionr.py

@@ -20,7 +20,6 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-
 import sys
 if sys.version_info[0] == 2 or sys.version_info[1] < 5:
     print('Error, Onionr requires Python 3.5+')
@@ -30,6 +29,7 @@ import webbrowser
 from threading import Thread
 import api, core, config, logger, onionrplugins as plugins, onionrevents as events
 import onionrutils
+import netcontroller
 from netcontroller import NetController
 from onionrblockapi import Block
 import onionrproofs, onionrexceptions, onionrusers
@@ -67,8 +67,12 @@ class Onionr:
         data_exists = Onionr.setupConfig(self.dataDir, self = self)
 
         self.onionrCore = core.Core()
+        #self.deleteRunFiles()
         self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore)
 
+        self.clientAPIInst = '' # Client http api instance
+        self.publicAPIInst = '' # Public http api instance
+
         # Handle commands
 
         self.debug = False # Whole application debugging
@@ -105,11 +109,12 @@ class Onionr:
         # Get configuration
         if type(config.get('client.webpassword')) is type(None):
             config.set('client.webpassword', base64.b16encode(os.urandom(32)).decode('utf-8'), savefile=True)
-        if type(config.get('client.port')) is type(None):
+        if type(config.get('client.client.port')) is type(None):
-            randomPort = 0
+            randomPort = netcontroller.getOpenPort()
-            while randomPort < 1024:
+            config.set('client.client.port', randomPort, savefile=True)
-                randomPort = self.onionrCore._crypto.secrets.randbelow(65535)
+        if type(config.get('client.public.port')) is type(None):
-            config.set('client.port', randomPort, savefile=True)
+            randomPort = netcontroller.getOpenPort()
+            config.set('client.public.port', randomPort, savefile=True)
         if type(config.get('client.participate')) is type(None):
             config.set('client.participate', True, savefile=True)
         if type(config.get('client.api_version')) is type(None):
@@ -176,6 +181,7 @@ class Onionr:
             'getfile': self.getFile,
 
             'listconn': self.listConn,
+            'list-conn': self.listConn,
 
             'import-blocks': self.onionrUtils.importNewBlocks,
             'importblocks': self.onionrUtils.importNewBlocks,
@@ -345,46 +351,9 @@ class Onionr:
                 except IndexError:
                     logger.error('Friend ID is required.')
                 except onionrexceptions.KeyNotKnown:
-                    logger.error('That peer is not in our database')
+                    self.onionrCore.addPeer(friend)
-                else:
-                    if action == 'add':
-                        friend.setTrust(1)
-                        logger.info('Added %s as friend.' % (friend.publicKey,))
-                    else:
-                        friend.setTrust(0)
-                        logger.info('Removed %s as friend.' % (friend.publicKey,))
-            else:
-                logger.info('Syntax: friend add/remove/list [address]')
-
-
-    def friendCmd(self):
-        '''List, add, or remove friend(s)
-        Changes their peer DB entry.
-        '''
-        friend = ''
-        try:
-            # Get the friend command
-            action = sys.argv[2]
-        except IndexError:
-            logger.info('Syntax: friend add/remove/list [address]')
-        else:
-            action = action.lower()
-            if action == 'list':
-                # List out peers marked as our friend
-                for friend in self.onionrCore.listPeers(randomOrder=False, trust=1):
-                    if friend == self.onionrCore._crypto.pubKey: # do not list our key
-                        continue
-                    friendProfile = onionrusers.OnionrUser(self.onionrCore, friend)
-                    logger.info(friend + ' - ' + friendProfile.getName())
-            elif action in ('add', 'remove'):
-                try:
-                    friend = sys.argv[3]
-                    if not self.onionrUtils.validatePubKey(friend):
-                        raise onionrexceptions.InvalidPubkey('Public key is invalid')
                     friend = onionrusers.OnionrUser(self.onionrCore, friend)
-                except IndexError:
+                finally:
-                    logger.error('Friend ID is required.')
-                else:
                     if action == 'add':
                         friend.setTrust(1)
                         logger.info('Added %s as friend.' % (friend.publicKey,))
@@ -394,6 +363,15 @@ class Onionr:
             else:
                 logger.info('Syntax: friend add/remove/list [address]')
 
+    def deleteRunFiles(self):
+        try:
+            os.remove(self.onionrCore.publicApiHostFile)
+        except FileNotFoundError:
+            pass
+        try:
+            os.remove(self.onionrCore.privateApiHostFile)
+        except FileNotFoundError:
+            pass
 
     def banBlock(self):
         try:
@@ -691,12 +669,18 @@ class Onionr:
                     os.remove('.onionr-lock')
                 except FileNotFoundError:
                     pass
+    def setClientAPIInst(self, inst):
+        self.clientAPIInst = inst
+
+    def getClientApi(self):
+        while self.clientAPIInst == '':
+            time.sleep(0.5)
+        return self.clientAPIInst
 
     def daemon(self):
         '''
             Starts the Onionr communication daemon
         '''
-
         communicatorDaemon = './communicator2.py'
 
         # remove runcheck if it exists
@@ -704,28 +688,28 @@ class Onionr:
             logger.debug('Runcheck file found on daemon start, deleting in advance.')
             os.remove('data/.runcheck')
 
-        apiThread = Thread(target = api.API, args = (self.debug, API_VERSION))
+        Thread(target=api.API, args=(self, self.debug, API_VERSION)).start()
-        apiThread.start()
+        Thread(target=api.PublicAPI, args=[self.getClientApi()]).start()
-
         try:
-            time.sleep(3)
+            time.sleep(0)
         except KeyboardInterrupt:
             logger.debug('Got keyboard interrupt, shutting down...')
             time.sleep(1)
             self.onionrUtils.localCommand('shutdown')
-        else:
+
-            apiHost = '127.0.0.1'
+        apiHost = ''
-            if apiThread.isAlive():
+        while apiHost == '':
             try:
-                    with open(self.onionrCore.dataDir + 'host.txt', 'r') as hostFile:
+                with open(self.onionrCore.publicApiHostFile, 'r') as hostFile:
                     apiHost = hostFile.read()
             except FileNotFoundError:
                 pass
+            time.sleep(0.5)
         Onionr.setupConfig('data/', self = self)
 
         if self._developmentMode:
             logger.warn('DEVELOPMENT MODE ENABLED (LESS SECURE)', timestamp = False)
-                net = NetController(config.get('client.port', 59496), apiServerIP=apiHost)
+        net = NetController(config.get('client.public.port', 59497), apiServerIP=apiHost)
         logger.debug('Tor is starting...')
         if not net.startTor():
             self.onionrUtils.localCommand('shutdown')
@@ -761,6 +745,9 @@ class Onionr:
         except KeyboardInterrupt:
             self.onionrCore.daemonQueueAdd('shutdown')
             self.onionrUtils.localCommand('shutdown')
+        time.sleep(3)
+        self.deleteRunFiles()
+        net.killTor()
         return
 
     def killDaemon(self):

onionr/onionrblockapi.py

@@ -245,7 +245,7 @@ class Block:
                         blockFile.write(self.getRaw().encode())
                 else:
                     self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire())
-                
+                if self.hash != False:
                     self.update()
 
                 return self.getHash()

onionr/onionrcrypto.py

@@ -269,7 +269,8 @@ class OnionrCrypto:
         except AttributeError:
             pass
         
-        difficulty = math.floor(dataLen / 1000000)
+        difficulty = onionrproofs.getDifficultyForNewBlock(blockContent, ourBlock=False)
+        
         if difficulty < int(config.get('general.minimum_block_pow')):
             difficulty = int(config.get('general.minimum_block_pow'))
         mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()

onionr/onionrdaemontools.py

@@ -45,7 +45,7 @@ class DaemonTools:
 
         ourID = self.daemon._core.hsAddress.strip()
 
-        url = 'http://' + peer + '/public/announce/'
+        url = 'http://' + peer + '/announce'
         data = {'node': ourID}
 
         combinedNodes = ourID + peer

onionr/onionrpeers.py

@@ -79,12 +79,14 @@ def peerCleanup(coreInst):
     logger.info('Cleaning peers...')
     config.reload()
 
-    minScore = int(config.get('peers.minimum_score', -100))
-    maxPeers = int(config.get('peers.max_stored', 5000))
-
     adders = getScoreSortedPeerList(coreInst)
     adders.reverse()
     
+    if len(adders) > 1:
+
+        minScore = int(config.get('peers.minimum_score', -100))
+        maxPeers = int(config.get('peers.max_stored', 5000))
+
         for address in adders:
             # Remove peers that go below the negative score
             if PeerProfiles(address, coreInst).score < minScore:

onionr/onionrproofs.py

@@ -19,7 +19,57 @@
 '''
 
 import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json
-import core, config
+import core, onionrutils, config
+import onionrblockapi
+
+def getDifficultyModifier(coreOrUtilsInst=None):
+    '''Accepts a core or utils instance returns 
+    the difficulty modifier for block storage based 
+    on a variety of factors, currently only disk use.
+    '''
+    classInst = coreOrUtilsInst
+    retData = 0
+    if isinstance(classInst, core.Core):
+        useFunc = classInst._utils.storageCounter.getPercent
+    elif isinstance(classInst, onionrutils.OnionrUtils):
+        useFunc = classInst.storageCounter.getPercent
+    else:
+        useFunc = core.Core()._utils.storageCounter.getPercent
+
+    percentUse = useFunc()
+
+    if percentUse >= 0.50:
+        retData += 1
+    elif percentUse >= 0.75:
+            retData += 2
+    elif percentUse >= 0.95:
+            retData += 3
+
+    return retData
+
+def getDifficultyForNewBlock(data, ourBlock=True):
+    '''
+    Get difficulty for block. Accepts size in integer, Block instance, or str/bytes full block contents
+    '''
+    retData = 0
+    dataSize = 0
+    if isinstance(data, onionrblockapi.Block):
+        dataSize = len(data.getRaw().encode('utf-8'))
+    elif isinstance(data, str):
+        dataSize = len(data.encode('utf-8'))
+    elif isinstance(data, bytes):
+        dataSize = len(data)
+    elif isinstance(data, int):
+        dataSize = data
+    else:
+        raise ValueError('not Block, str, or int')
+    if ourBlock:
+        minDifficulty = config.get('general.minimum_send_pow')
+    else:
+        minDifficulty = config.get('general.minimum_block_pow')
+
+    retData = max(minDifficulty, math.floor(dataSize / 1000000)) + getDifficultyModifier()
+    return retData
 
 def getHashDifficulty(h):
     '''
@@ -55,6 +105,7 @@ class DataPOW:
         self.difficulty = 0
         self.data = data
         self.threadCount = threadCount
+        self.rounds = 0
         config.reload()
 
         if forceDifficulty == 0:
@@ -96,6 +147,7 @@ class DataPOW:
         while self.hashing:
             rand = nacl.utils.random()
             token = nacl.hash.blake2b(rand + self.data).decode()
+            self.rounds += 1
             #print(token)
             if self.puzzle == token[0:self.difficulty]:
                 self.hashing = False
@@ -106,6 +158,7 @@ class DataPOW:
             endTime = math.floor(time.time())
             if self.reporting:
                 logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True)
+                logger.debug('Round count: %s' % (self.rounds,))
             self.result = (token, rand)
 
     def shutdown(self):
@@ -146,17 +199,27 @@ class DataPOW:
         return result
 
 class POW:
-    def __init__(self, metadata, data, threadCount = 5):
+    def __init__(self, metadata, data, threadCount = 5, forceDifficulty=0, coreInst=None):
         self.foundHash = False
         self.difficulty = 0
         self.data = data
         self.metadata = metadata
         self.threadCount = threadCount
 
+        try:
+            assert isinstance(coreInst, core.Core)
+        except AssertionError:
+            myCore = core.Core()
+        else:
+            myCore = coreInst
+
         dataLen = len(data) + len(json.dumps(metadata))
-        self.difficulty = math.floor(dataLen / 1000000)
+
-        if self.difficulty <= 2:
+        if forceDifficulty > 0:
-            self.difficulty = int(config.get('general.minimum_block_pow'))
+            self.difficulty = forceDifficulty
+        else:
+            # Calculate difficulty. Dumb for now, may use good algorithm in the future.
+            self.difficulty = getDifficultyForNewBlock(dataLen)
             
         try:
             self.data = self.data.encode()
@@ -168,7 +231,6 @@ class POW:
         self.mainHash = '0' * 64
         self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))]
 
-        myCore = core.Core()
         for i in range(max(1, threadCount)):
             t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore))
             t.start()

onionr/onionrutils.py

@@ -155,20 +155,26 @@ class OnionrUtils:
         '''
             Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers.
         '''
-
         config.reload()
         self.getTimeBypassToken()
         # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
+        hostname = ''
+        maxWait = 5
+        waited = 0
+        while hostname == '':
             try:
-            with open(self._core.dataDir + 'host.txt', 'r') as host:
+                with open(self._core.privateApiHostFile, 'r') as host:
                     hostname = host.read()
             except FileNotFoundError:
+                time.sleep(1)
+                waited += 1
+                if waited == maxWait:
                     return False
-        payload = 'http://%s:%s/client/?action=%s&token=%s&timingToken=%s' % (hostname, config.get('client.port'), command, config.get('client.webpassword'), self.timingToken)
         if data != '':
-            payload += '&data=' + urllib.parse.quote_plus(data)
+            data = '&data=' + urllib.parse.quote_plus(data)
+        payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data)
         try:
-            retData = requests.get(payload).text
+            retData = requests.get(payload, headers={'token': config.get('client.webpassword')}).text
         except Exception as error:
             if not silent:
                 logger.error('Failed to make local request (command: %s):%s' % (command, error))

onionr/static-data/default_config.json

@@ -1,9 +1,9 @@
 {
     "general" : {
         "dev_mode" : true,
-        "display_header" : true,
+        "display_header" : false,
-        "minimum_block_pow": 5,
+        "minimum_block_pow": 3,
-        "minimum_send_pow": 5,
+        "minimum_send_pow": 3,
         "socket_servers": false,
         "security_level": 0,
         "max_block_age": 2678400,
@@ -69,7 +69,7 @@
      },
 
     "allocations" : {
-        "disk" : 10000000000,
+        "disk" : 100000000,
         "net_total" : 1000000000,
         "blockCache" : 5000000,
         "blockCacheTotal" : 50000000

onionr/static-data/index.html

@@ -1,7 +1,7 @@
 <h1>This is an Onionr Node</h1>
 
-<p>The content on this server is not necessarily created by the server owner, and was not necessarily stored specifically with the owner's knowledge of its contents.</p>
+<p>The content on this server was not necessarily intentionally stored or created by the owner(s) of this server.</p>
 
-<p>Onionr is a decentralized data storage system that anyone can insert data into.</p>
+<p>Onionr is a decentralized peer-to-peer data storage system.</p>
 
 <p>To learn more about Onionr, see the website at <a href="https://onionr.voidnet.tech/">https://Onionr.VoidNet.tech/</a></p>

onionr/static-data/www/board/board.js

@@ -0,0 +1,58 @@
+webpassword = ''
+requested = []
+
+document.getElementById('webpassWindow').style.display = 'block';
+
+var windowHeight = window.innerHeight;
+document.getElementById('webpassWindow').style.height = windowHeight + "px";
+
+function httpGet(theUrl) {
+    var xmlHttp = new XMLHttpRequest()
+    xmlHttp.open( "GET", theUrl, false ) // false for synchronous request
+    xmlHttp.setRequestHeader('token', webpassword)
+    xmlHttp.send( null )
+    if (xmlHttp.status == 200){
+        return xmlHttp.responseText
+    }
+    else{
+        return "";
+    }
+}
+function appendMessages(msg){
+    el = document.createElement('div')
+    el.className = 'entry'
+    el.innerText = msg
+    document.getElementById('feed').appendChild(el)
+    document.getElementById('feed').appendChild(document.createElement('br'))
+}
+
+function getBlocks(){
+    if (document.getElementById('none') !== null){
+        document.getElementById('none').remove();
+
+    }
+    var feedText =  httpGet('/getblocksbytype/txt')
+    var blockList = feedText.split(',')
+    for (i = 0; i < blockList.length; i++){
+        if (! requested.includes(blockList[i])){
+            bl = httpGet('/gethtmlsafeblockdata/' + blockList[i])
+            appendMessages(bl)
+            requested.push(blockList[i])
+            }
+        }
+}
+
+document.getElementById('registerPassword').onclick = function(){
+    webpassword = document.getElementById('webpassword').value
+    if (httpGet('/ping') === 'pong!'){
+        document.getElementById('webpassWindow').style.display = 'none'
+        getBlocks()
+    }
+    else{
+        alert('Sorry, but that password appears invalid.')
+    }
+}
+
+document.getElementById('refreshFeed').onclick = function(){
+    getBlocks()
+}

onionr/static-data/www/board/index.html

@@ -0,0 +1,21 @@
+<!DOCTYPE HTML>
+<html>
+    <head>
+        <meta charset='utf-8'>
+        <title>
+            OnionrBoard
+        </title>
+        <link rel='stylesheet' href='theme.css'>
+    </head>
+    <body>    
+        <div id='webpassWindow' class='hidden'>
+                <p>Welcome to OnionrBoard</p>
+                <p>Please enter the webpassword. You can get this from running the 'details' command in Onionr.</p>
+                <input id='webpassword' type='password' placeholder="Web password for daemon" value='7AF13568657CE63D6DB7E686BF05537D36598ED739B21E3F023E3FD3DEA2FC8F'>
+                <button id='registerPassword'>Unlock Onionr</button>
+        </div>
+        <input type='button' id='refreshFeed' value='Refresh Feed'>
+        <div id='feed'><span id='none'>None Yet :)</span></div>
+        <script src='board.js'></script>
+    </body>
+</html>

onionr/static-data/www/board/theme.css

@@ -0,0 +1,31 @@
+h1, h2, h3{
+    font-family: sans-serif;
+}
+.hidden{
+    display: none;
+}
+p{
+    font-family: sans-serif;
+}
+#webpassWindow{
+    background-color: black;
+    border: 1px solid black;
+    border-radius: 5px;
+    width: 100%;
+    z-index: 2;
+    color: white;
+    text-align: center;
+}
+
+.entry{
+    color: red;
+}
+
+#feed{
+    margin-left: 2%;
+    margin-right: 25%;
+    margin-top: 1em;
+    border: 2px solid black;
+    padding: 5px;
+    min-height: 50px;
+}

onionr/storagecounter.py

@@ -44,6 +44,11 @@ class StorageCounter:
             pass
         return retData
     
+    def getPercent(self):
+        '''Return percent (decimal/float) of disk space we're using'''
+        amount = self.getAmount()
+        return round(amount / self._core.config.get('allocations.disk'), 2)
+
     def addBytes(self, amount):
         '''Record that we are now using more disk space, unless doing so would exceed configured max'''
         newAmount = amount + self.getAmount()

requirements.txt

@@ -2,7 +2,6 @@ urllib3==1.23
 requests==2.20.0
 PyNaCl==1.2.1
 gevent==1.3.6
-sha3==0.2.1
 defusedxml==0.5.0
 Flask==1.0.2
 PySocks==1.6.8