+ Reformatted API, more efficient, standard, and secure now

* Various bug fixes

onionr/api.py

@@ -25,8 +25,6 @@ import core
 from onionrblockapi import Block
 import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionr
 
-API_VERSION = 0
-
 def guessMime(path):
     '''
         Guesses the mime type of a file from the input filename
@@ -67,6 +65,7 @@ class PublicAPI:
         self.torAdder = clientAPI._core.hsAddress
         self.i2pAdder = clientAPI._core.i2pAddress
         self.bindPort = config.get('client.public.port')
+        logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
 
         @app.before_request
         def validateRequest():
@@ -84,7 +83,7 @@ class PublicAPI:
             resp.headers["Content-Security-Policy"] =  "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
             resp.headers['X-Frame-Options'] = 'deny'
             resp.headers['X-Content-Type-Options'] = "nosniff"
-            resp.headers['X-API'] = API_VERSION
+            resp.headers['X-API'] = onionr.API_VERSION
             return resp
 
         @app.route('/')
@@ -105,8 +104,9 @@ class PublicAPI:
             return Response('\n'.join(bList))
 
         @app.route('/getdata/<name>')
-        def getBlockData():
+        def getBlockData(name):
             resp = ''
+            data = name
             if clientAPI._utils.validateHash(data):
                 if data not in self.hideBlocks:
                     if os.path.exists(clientAPI._core.dataDir + 'blocks/' + data + '.dat'):
@@ -117,6 +117,12 @@ class PublicAPI:
                 resp = ""
             return Response(resp)
 
+        @app.route('/www/<path:path>')
+        def wwwPublic(path):
+            if not config.get("www.public.run", True):
+                abort(403)
+            return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
+
         @app.route('/ping')
         def ping():
             return Response("pong!")
@@ -127,10 +133,48 @@ class PublicAPI:
 
         @app.route('/pex')
         def peerExchange():
-            response = ','.join(self._core.listAdders())
+            response = ','.join(clientAPI._core.listAdders())
             if len(response) == 0:
                 response = 'none'
-            resp = Response(response)
+            return Response(response)
+        
+        @app.route('/announce', methods=['post'])
+        def acceptAnnounce():
+            resp = 'failure'
+            powHash = ''
+            randomData = ''
+            newNode = ''
+            ourAdder = clientAPI._core.hsAddress.encode()
+            try:
+                newNode = request.form['node'].encode()
+            except KeyError:
+                logger.warn('No block specified for upload')
+                pass
+            else:
+                try:
+                    randomData = request.form['random']
+                    randomData = base64.b64decode(randomData)
+                except KeyError:
+                    logger.warn('No random data specified for upload')
+                else:
+                    nodes = newNode + clientAPI._core.hsAddress.encode()
+                    nodes = clientAPI._core._crypto.blake2bHash(nodes)
+                    powHash = clientAPI._core._crypto.blake2bHash(randomData + nodes)
+                    try:
+                        powHash = powHash.decode()
+                    except AttributeError:
+                        pass
+                    if powHash.startswith('0000'):
+                        try:
+                            newNode = newNode.decode()
+                        except AttributeError:
+                            pass
+                        if clientAPI._core.addAddress(newNode):
+                            resp = 'Success'
+                    else:
+                        logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)
+            resp = Response(resp)
+            return resp
 
         @app.route('/upload', methods=['post'])
         def upload():
@@ -156,6 +200,10 @@ class PublicAPI:
             return resp
 
         clientAPI.setPublicAPIInstance(self)
+        while self.torAdder == '':
+            clientAPI._core.refreshFirstStartVars()
+            self.torAdder = clientAPI._core.hsAddress
+            time.sleep(1)
         self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None)
         self.httpServer.serve_forever()
 
@@ -166,7 +214,7 @@ class API:
 
     callbacks = {'public' : {}, 'private' : {}}
 
-    def __init__(self, debug, API_VERSION):
+    def __init__(self, onionrInst, debug, API_VERSION):
         '''
             Initialize the api server, preping variables for later use
 
@@ -190,10 +238,11 @@ class API:
         self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
 
         self.publicAPI = None # gets set when the thread calls our setter... bad hack but kinda necessary with flask
-        threading.Thread(target=PublicAPI, args=(self,)).start()
+        #threading.Thread(target=PublicAPI, args=(self,)).start()
         self.host = setBindIP(self._core.privateApiHostFile)
         logger.info('Running api on %s:%s' % (self.host, self.bindPort))
         self.httpServer = ''
+        onionrInst.setClientAPIInst(self)
 
         @app.before_request
         def validateRequest():
@@ -211,14 +260,14 @@ class API:
             resp.headers["Content-Security-Policy"] =  "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
             resp.headers['X-Frame-Options'] = 'deny'
             resp.headers['X-Content-Type-Options'] = "nosniff"
-            resp.headers['X-API'] = API_VERSION
+            resp.headers['X-API'] = onionr.API_VERSION
-            resp.headers['Server'] = 'nginx'
+            resp.headers['Server'] = ''
             resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch.
             return resp
 
         @app.route('/ping')
         def ping():
-            return Respose("pong!")
+            return Response("pong!")
 
         @app.route('/')
         def hello():

onionr/communicator2.py

@@ -19,11 +19,10 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-#import gevent.monkey
-#gevent.monkey.patch_all()
 import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid
 import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
 import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs
+import binascii
 from dependencies import secrets
 from defusedxml import minidom
 
@@ -103,6 +102,7 @@ class OnionrCommunicatorDaemon:
         OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True)
         OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1)
         OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1)
+        OnionrCommunicatorTimers(self, self.detectAPICrash, 5, maxThreads=1)
         deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1)
 
         netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)
@@ -232,7 +232,10 @@ class OnionrCommunicatorDaemon:
                     content = content.encode()
                 except AttributeError:
                     pass
-                content = base64.b64decode(content) # content is base64 encoded in transport
+                try:
+                    content = base64.b64decode(content) # content is base64 encoded in transport
+                except binascii.Error:
+                    pass
                 realHash = self._core._crypto.sha3Hash(content)
                 try:
                     realHash = realHash.decode() # bytes on some versions for some reason
@@ -423,7 +426,7 @@ class OnionrCommunicatorDaemon:
         if len(peer) == 0:
             return False
         #logger.debug('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort))
-        url = 'http://' + peer + '/' + action
+        url = 'http://%s/%s' % (peer, action)
         if len(data) > 0:
             url += '&data=' + data
 
@@ -546,9 +549,9 @@ class OnionrCommunicatorDaemon:
 
     def detectAPICrash(self):
         '''exit if the api server crashes/stops'''
-        if self._core._utils.localCommand('ping', silent=False) != 'pong':
+        if self._core._utils.localCommand('ping', silent=False) not in ('pong', 'pong!'):
             for i in range(5):
-                if self._core._utils.localCommand('ping') == 'pong':
+                if self._core._utils.localCommand('ping') in ('pong', 'pong!'):
                     break # break for loop
                 time.sleep(1)
             else:

onionr/core.py

@@ -154,7 +154,7 @@ class Core:
 
         if address == config.get('i2p.ownAddr', None) or address == self.hsAddress:
             return False
-        if type(address) is type(None) or len(address) == 0:
+        if type(address) is None or len(address) == 0:
             return False
         if self._utils.validateID(address):
             conn = sqlite3.connect(self.addressDB, timeout=10)

onionr/netcontroller.py

@@ -77,7 +77,7 @@ class NetController:
         hsVer = '# v2 onions'
         if config.get('tor.v3onions'):
             hsVer = 'HiddenServiceVersion 3'
-            logger.debug('Using v3 onions :)')
+            logger.debug('Using v3 onions')
 
         if os.path.exists(self.torConfigLocation):
             os.remove(self.torConfigLocation)

onionr/onionr.py

@@ -67,6 +67,7 @@ class Onionr:
         data_exists = Onionr.setupConfig(self.dataDir, self = self)
 
         self.onionrCore = core.Core()
+        #self.deleteRunFiles()
         self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore)
 
         self.clientAPIInst = '' # Client http api instance
@@ -399,6 +400,16 @@ class Onionr:
                 logger.info('Syntax: friend add/remove/list [address]')
 
 
+    def deleteRunFiles(self):
+        try:
+            os.remove(self.onionrCore.publicApiHostFile)
+        except FileNotFoundError:
+            pass
+        try:
+            os.remove(self.onionrCore.privateApiHostFile)
+        except FileNotFoundError:
+            pass
+
     def banBlock(self):
         try:
             ban = sys.argv[2]
@@ -695,6 +706,13 @@ class Onionr:
                     os.remove('.onionr-lock')
                 except FileNotFoundError:
                     pass
+    def setClientAPIInst(self, inst):
+        self.clientAPIInst = inst
+
+    def getClientApi(self):
+        while self.clientAPIInst == '':
+            time.sleep(0.5)
+        return self.clientAPIInst
 
     def daemon(self):
         '''
@@ -708,64 +726,66 @@ class Onionr:
             logger.debug('Runcheck file found on daemon start, deleting in advance.')
             os.remove('data/.runcheck')
 
-        apiTarget = api.API
+        Thread(target=api.API, args=(self, self.debug, API_VERSION)).start()
-        apiThread = Thread(target = apiTarget, args = (self.debug, API_VERSION))
+        Thread(target=api.PublicAPI, args=[self.getClientApi()]).start()
-        apiThread.start()
-
         try:
-            time.sleep(3)
+            time.sleep(0)
         except KeyboardInterrupt:
             logger.debug('Got keyboard interrupt, shutting down...')
             time.sleep(1)
             self.onionrUtils.localCommand('shutdown')
+
+        apiHost = ''
+        while apiHost == '':
+            try:
+                with open(self.onionrCore.publicApiHostFile, 'r') as hostFile:
+                    apiHost = hostFile.read()
+            except FileNotFoundError:
+                pass
+            time.sleep(0.5)
+        Onionr.setupConfig('data/', self = self)
+
+        if self._developmentMode:
+            logger.warn('DEVELOPMENT MODE ENABLED (LESS SECURE)', timestamp = False)
+        net = NetController(config.get('client.public.port', 59497), apiServerIP=apiHost)
+        logger.debug('Tor is starting...')
+        if not net.startTor():
+            self.onionrUtils.localCommand('shutdown')
+            sys.exit(1)
+        if len(net.myID) > 0 and config.get('general.security_level') == 0:
+            logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID))
         else:
-            apiHost = '127.0.0.1'
+            logger.debug('.onion service disabled')
-            if apiThread.isAlive():
+        logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey))
-                try:
+        time.sleep(1)
-                    with open(self.onionrCore.publicApiHostFile, 'r') as hostFile:
-                        apiHost = hostFile.read()
-                except FileNotFoundError:
-                    pass
-                Onionr.setupConfig('data/', self = self)
 
-                if self._developmentMode:
+        # TODO: make runable on windows
-                    logger.warn('DEVELOPMENT MODE ENABLED (LESS SECURE)', timestamp = False)
+        communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)])
-                net = NetController(config.get('client.public.port', 59497), apiServerIP=apiHost)
-                logger.debug('Tor is starting...')
-                if not net.startTor():
-                    self.onionrUtils.localCommand('shutdown')
-                    sys.exit(1)
-                if len(net.myID) > 0 and config.get('general.security_level') == 0:
-                    logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID))
-                else:
-                    logger.debug('.onion service disabled')
-                logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey))
-                time.sleep(1)
 
-                # TODO: make runable on windows
+        # print nice header thing :)
-                communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)])
+        if config.get('general.display_header', True):
+            self.header()
 
-                # print nice header thing :)
+        # print out debug info
-                if config.get('general.display_header', True):
+        self.version(verbosity = 5, function = logger.debug)
-                    self.header()
+        logger.debug('Python version %s' % platform.python_version())
 
-                # print out debug info
+        logger.debug('Started communicator.')
-                self.version(verbosity = 5, function = logger.debug)
-                logger.debug('Python version %s' % platform.python_version())
 
-                logger.debug('Started communicator.')
+        events.event('daemon_start', onionr = self)
+        try:
+            while True:
+                time.sleep(5)
 
-                events.event('daemon_start', onionr = self)
+                # Break if communicator process ends, so we don't have left over processes
-                try:
+                if communicatorProc.poll() is not None:
-                    while True:
+                    break
-                        time.sleep(5)
+        except KeyboardInterrupt:
-
+            self.onionrCore.daemonQueueAdd('shutdown')
-                        # Break if communicator process ends, so we don't have left over processes
+            self.onionrUtils.localCommand('shutdown')
-                        if communicatorProc.poll() is not None:
+        time.sleep(3)
-                            break
+        self.deleteRunFiles()
-                except KeyboardInterrupt:
+        net.killTor()
-                    self.onionrCore.daemonQueueAdd('shutdown')
-                    self.onionrUtils.localCommand('shutdown')
         return
 
     def killDaemon(self):

onionr/onionrdaemontools.py

@@ -45,7 +45,7 @@ class DaemonTools:
 
         ourID = self.daemon._core.hsAddress.strip()
 
-        url = 'http://' + peer + '/public/announce/'
+        url = 'http://' + peer + '/announce'
         data = {'node': ourID}
 
         combinedNodes = ourID + peer

onionr/onionrpeers.py

@@ -79,27 +79,29 @@ def peerCleanup(coreInst):
     logger.info('Cleaning peers...')
     config.reload()
 
-    minScore = int(config.get('peers.minimum_score', -100))
-    maxPeers = int(config.get('peers.max_stored', 5000))
-
     adders = getScoreSortedPeerList(coreInst)
     adders.reverse()
     
-    for address in adders:
+    if len(adders) > 1:
-        # Remove peers that go below the negative score
+
-        if PeerProfiles(address, coreInst).score < minScore:
+        minScore = int(config.get('peers.minimum_score', -100))
-            coreInst.removeAddress(address)
+        maxPeers = int(config.get('peers.max_stored', 5000))
-            try:
+
-                if (int(coreInst._utils.getEpoch()) - int(coreInst.getPeerInfo(address, 'dateSeen'))) >= 600:
+        for address in adders:
-                    expireTime = 600
+            # Remove peers that go below the negative score
-                else:
+            if PeerProfiles(address, coreInst).score < minScore:
-                    expireTime = 86400
+                coreInst.removeAddress(address)
-                coreInst._blacklist.addToDB(address, dataType=1, expire=expireTime)
+                try:
-            except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue
+                    if (int(coreInst._utils.getEpoch()) - int(coreInst.getPeerInfo(address, 'dateSeen'))) >= 600:
-                pass
+                        expireTime = 600
-            except ValueError:
+                    else:
-                pass
+                        expireTime = 86400
-            logger.warn('Removed address ' + address + '.')
+                    coreInst._blacklist.addToDB(address, dataType=1, expire=expireTime)
+                except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue
+                    pass
+                except ValueError:
+                    pass
+                logger.warn('Removed address ' + address + '.')
 
     # Unban probably not malicious peers TODO improve
     coreInst._blacklist.deleteExpired(dataType=1)

onionr/onionrproofs.py

@@ -238,7 +238,6 @@ class POW:
                     break
                 else:
                     time.sleep(2)
-                print('boi')
         except KeyboardInterrupt:
             self.shutdown()
             logger.warn('Got keyboard interrupt while waiting for POW result, stopping')

onionr/onionrutils.py

@@ -159,17 +159,17 @@ class OnionrUtils:
         config.reload()
         self.getTimeBypassToken()
         # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
-        try:
+        hostname = ''
-            with open(self._core.privateApiHostFile, 'r') as host:
+        while hostname == '':
-                hostname = host.read()
+            try:
-        except FileNotFoundError:
+                with open(self._core.privateApiHostFile, 'r') as host:
-            return False
+                    hostname = host.read()
+            except FileNotFoundError:
+                print('wat')
+                time.sleep(1)
         if data != '':
             data = '&data=' + urllib.parse.quote_plus(data)
         payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data)
-        #payload = 'http://%s:%s/client/?action=%s&token=%s&timingToken=%s' % (hostname, config.get('client.client.port'), command, config.get('client.webpassword'), self.timingToken)
-        #if data != '':
-        #    payload += '&data=' + urllib.parse.quote_plus(data)
         try:
             retData = requests.get(payload, headers={'token': config.get('client.webpassword')}).text
         except Exception as error:

onionr/static-data/index.html

@@ -1,7 +1,7 @@
 <h1>This is an Onionr Node</h1>
 
-<p>The content on this server is not necessarily created by the server owner, and was not necessarily stored specifically with the owner's knowledge of its contents.</p>
+<p>The content on this server was not necessarily intentionally stored or created by the owner(s) of this server.</p>
 
-<p>Onionr is a decentralized data storage system that anyone can insert data into.</p>
+<p>Onionr is a decentralized peer-to-peer data storage system.</p>
 
 <p>To learn more about Onionr, see the website at <a href="https://onionr.voidnet.tech/">https://Onionr.VoidNet.tech/</a></p>

requirements.txt

@@ -2,7 +2,6 @@ urllib3==1.23
 requests==2.20.0
 PyNaCl==1.2.1
 gevent==1.3.6
-sha3==0.2.1
 defusedxml==0.5.0
 Flask==1.0.2
 PySocks==1.6.8