refactored api servers, especially the public server. Client server to recieve similar refactoring soon

onionr/api.py

@@ -17,61 +17,20 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import random, threading, hmac, base64, time, os, json, socket
+import threading, hmac, base64, time, os
-from gevent.pywsgi import WSGIServer, WSGIHandler
+from gevent.pywsgi import WSGIServer
-from gevent import Timeout
 import flask
 from flask import request, Response, abort, send_from_directory
 import core
-import onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionrblockapi
+import onionrexceptions, onionrcrypto, logger, config
 import httpapi
 from httpapi import friendsapi, profilesapi, configapi, miscpublicapi, miscclientapi, insertblock, onionrsitesapi
 from onionrservices import httpheaders
 import onionr
 from onionrutils import bytesconverter, stringvalidators, epoch, mnemonickeys
-from httpapi import apiutils
+from httpapi import apiutils, security, fdsafehandler
 
 config.reload()
-class FDSafeHandler(WSGIHandler):
-    '''Our WSGI handler. Doesn't do much non-default except timeouts'''
-    def handle(self):
-        self.timeout = Timeout(120, Exception)
-        self.timeout.start()
-        try:
-            WSGIHandler.handle(self)
-        except Exception:
-            self.handle_error()
-        finally:
-            self.timeout.close()
-
-    def handle_error(self):
-        if v is self.timeout:
-            self.result = [b"Timeout"]
-            self.start_response("200 OK", [])
-            self.process_result()
-        else:
-            WSGIHandler.handle_error(self) 
-
-def setBindIP(filePath=''):
-    '''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)'''
-    if config.get('general.random_bind_ip', True):
-        hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
-        data = '.'.join(hostOctets)
-        # Try to bind IP. Some platforms like Mac block non normal 127.x.x.x
-        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        try:
-            s.bind((data, 0))
-        except OSError:
-            # if mac/non-bindable, show warning and default to 127.0.0.1
-            logger.warn('Your platform appears to not support random local host addresses 127.x.x.x. Falling back to 127.0.0.1.')
-            data = '127.0.0.1'
-        s.close()
-    else:
-        data = '127.0.0.1'
-    if filePath != '':
-        with open(filePath, 'w') as bindFile:
-            bindFile.write(data)
-    return data
 
 class PublicAPI:
     '''
@@ -82,94 +41,27 @@ class PublicAPI:
         app = flask.Flask('PublicAPI')
         self.i2pEnabled = config.get('i2p.host', False)
         self.hideBlocks = [] # Blocks to be denied sharing
-        self.host = setBindIP(clientAPI._core.publicApiHostFile)
+        self.host = apiutils.setbindip.set_bind_IP(clientAPI._core.publicApiHostFile, clientAPI._core)
         self.torAdder = clientAPI._core.hsAddress
         self.i2pAdder = clientAPI._core.i2pAddress
         self.bindPort = config.get('client.public.port')
         self.lastRequest = 0
         self.hitCount = 0 # total rec requests to public api since server started
+        self.config = config
+        self.clientAPI = clientAPI
+        self.API_VERSION = onionr.API_VERSION
         logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
 
-        @app.before_request
-        def validateRequest():
-            '''Validate request has the correct hostname'''
-            # If high security level, deny requests to public (HS should be disabled anyway for Tor, but might not be for I2P)
-            if config.get('general.security_level', default=1) > 0:
-                abort(403)
-            if type(self.torAdder) is None and type(self.i2pAdder) is None:
-                # abort if our hs addresses are not known
-                abort(403)
-            if request.host not in (self.i2pAdder, self.torAdder):
-                # Disallow connection if wrong HTTP hostname, in order to prevent DNS rebinding attacks
-                abort(403)
-            self.hitCount += 1 # raise hit count for valid requests
-
-        @app.after_request
-        def sendHeaders(resp):
-            '''Send api, access control headers'''
-            resp = httpheaders.set_default_onionr_http_headers(resp)
-            # Network API version
-            resp.headers['X-API'] = onionr.API_VERSION
-            self.lastRequest = epoch.get_rounded_epoch(roundS=5)
-            return resp
-
-        @app.route('/')
-        def banner():
-            # Display a bit of information to people who visit a node address in their browser
-            try:
-                with open('static-data/index.html', 'r') as html:
-                    resp = Response(html.read(), mimetype='text/html')
-            except FileNotFoundError:
-                resp = Response("")
-            return resp
-
-        @app.route('/getblocklist')
-        def getBlockList():
-            return httpapi.miscpublicapi.public_block_list(clientAPI, self, request)
-
-        @app.route('/getdata/<name>')
-        def getBlockData(name):
-            # Share data for a block if we have it
-            return httpapi.miscpublicapi.public_get_block_data(clientAPI, self, name)
-
-        @app.route('/www/<path:path>')
-        def wwwPublic(path):
-            # A way to share files directly over your .onion
-            if not config.get("www.public.run", True):
-                abort(403)
-            return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
-
-        @app.route('/ping')
-        def ping():
-            # Endpoint to test if nodes are up
-            return Response("pong!")
-
-        @app.route('/pex')
-        def peerExchange():
-            response = ','.join(clientAPI._core.listAdders(recent=3600))
-            if len(response) == 0:
-                response = ''
-            return Response(response)
-        
-        @app.route('/announce', methods=['post'])
-        def acceptAnnounce():
-            resp = httpapi.miscpublicapi.announce(clientAPI, request)
-            return resp
-
-        @app.route('/upload', methods=['post'])
-        def upload():
-            '''Accept file uploads. In the future this will be done more often than on creation 
-            to speed up block sync
-            '''
-            return httpapi.miscpublicapi.upload(clientAPI, request)
-
         # Set instances, then startup our public api server
         clientAPI.setPublicAPIInstance(self)
         while self.torAdder == '':
             clientAPI._core.refreshFirstStartVars()
             self.torAdder = clientAPI._core.hsAddress
             time.sleep(0.1)
-        self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None, handler_class=FDSafeHandler)
+        
+        app.register_blueprint(security.public.PublicAPISecurity(self).public_api_security_bp)
+        app.register_blueprint(miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp)
+        self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None, handler_class=fdsafehandler.FDSafeHandler)
         self.httpServer.serve_forever()
 
 class API:
@@ -200,12 +92,13 @@ class API:
 
         self.publicAPI = None # gets set when the thread calls our setter... bad hack but kinda necessary with flask
         #threading.Thread(target=PublicAPI, args=(self,)).start()
-        self.host = setBindIP(self._core.privateApiHostFile)
+        self.host = apiutils.setbindip.set_bind_IP(self._core.privateApiHostFile, self._core)
         logger.info('Running api on %s:%s' % (self.host, self.bindPort))
         self.httpServer = ''
 
         self.queueResponse = {}
         onionrInst.setClientAPIInst(self)
+        app.register_blueprint(security.client.ClientAPISecurity(self).client_api_security_bp)
         app.register_blueprint(friendsapi.friends)
         app.register_blueprint(profilesapi.profile_BP)
         app.register_blueprint(configapi.config_BP)
@@ -303,7 +196,7 @@ class API:
         def getHumanReadable(name):
             return Response(mnemonickeys.get_human_readable_ID(name))
 
-        self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=FDSafeHandler)
+        self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=fdsafehandler.FDSafeHandler)
         self.httpServer.serve_forever()
 
     def setPublicAPIInstance(self, inst):

onionr/communicatorutils/lookupadders.py

@@ -21,7 +21,7 @@ import logger
 from onionrutils import stringvalidators
 
 def lookup_new_peer_transports_with_communicator(comm_inst):
-    logger.info('Looking up new addresses...', terminal=True)
+    logger.info('Looking up new addresses...')
     tryAmount = 1
     newPeers = []
     for i in range(tryAmount):

onionr/httpapi/apiutils/__init__.py

@@ -1,44 +1,3 @@
-import json
+from . import shutdown, setbindip, getblockdata
-import core, onionrblockapi
-from onionrutils import bytesconverter, stringvalidators
-from . import shutdown
 
-class GetBlockData:
+GetBlockData = getblockdata.GetBlockData
-    def __init__(self, client_api_inst=None):
-        if client_api_inst is None:
-            self.client_api_inst = None
-            self.c = core.Core()
-        elif isinstance(client_api_inst, core.Core):
-            self.client_api_inst = None
-            self.c = client_api_inst
-        else:
-            self.client_api_Inst = client_api_inst
-            self.c = core.Core()
-    
-    def get_block_data(self, bHash, decrypt=False, raw=False, headerOnly=False):
-        assert stringvalidators.validate_hash(bHash)
-        bl = onionrblockapi.Block(bHash, core=self.c)
-        if decrypt:
-            bl.decrypt()
-            if bl.isEncrypted and not bl.decrypted:
-                raise ValueError
-
-        if not raw:
-            if not headerOnly:
-                retData = {'meta':bl.bheader, 'metadata': bl.bmetadata, 'content': bl.bcontent}
-                for x in list(retData.keys()):
-                    try:
-                        retData[x] = retData[x].decode()
-                    except AttributeError:
-                        pass
-            else:
-                validSig = False
-                signer = bytesconverter.bytes_to_str(bl.signer)
-                if bl.isSigned() and stringvalidators.validate_pub_key(signer) and bl.isSigner(signer):
-                    validSig = True                    
-                bl.bheader['validSig'] = validSig
-                bl.bheader['meta'] = ''
-                retData = {'meta': bl.bheader, 'metadata': bl.bmetadata}
-            return json.dumps(retData)
-        else:
-            return bl.raw

onionr/httpapi/apiutils/getblockdata.py

@@ -0,0 +1,42 @@
+import json
+import core, onionrblockapi
+from onionrutils import bytesconverter, stringvalidators
+class GetBlockData:
+    def __init__(self, client_api_inst=None):
+        if client_api_inst is None:
+            self.client_api_inst = None
+            self.c = core.Core()
+        elif isinstance(client_api_inst, core.Core):
+            self.client_api_inst = None
+            self.c = client_api_inst
+        else:
+            self.client_api_Inst = client_api_inst
+            self.c = core.Core()
+    
+    def get_block_data(self, bHash, decrypt=False, raw=False, headerOnly=False):
+        assert stringvalidators.validate_hash(bHash)
+        bl = onionrblockapi.Block(bHash, core=self.c)
+        if decrypt:
+            bl.decrypt()
+            if bl.isEncrypted and not bl.decrypted:
+                raise ValueError
+
+        if not raw:
+            if not headerOnly:
+                retData = {'meta':bl.bheader, 'metadata': bl.bmetadata, 'content': bl.bcontent}
+                for x in list(retData.keys()):
+                    try:
+                        retData[x] = retData[x].decode()
+                    except AttributeError:
+                        pass
+            else:
+                validSig = False
+                signer = bytesconverter.bytes_to_str(bl.signer)
+                if bl.isSigned() and stringvalidators.validate_pub_key(signer) and bl.isSigner(signer):
+                    validSig = True                    
+                bl.bheader['validSig'] = validSig
+                bl.bheader['meta'] = ''
+                retData = {'meta': bl.bheader, 'metadata': bl.bmetadata}
+            return json.dumps(retData)
+        else:
+            return bl.raw

onionr/httpapi/apiutils/setbindip.py

@@ -0,0 +1,25 @@
+import random, socket
+import config, logger
+def set_bind_IP(filePath='', core_inst=None):
+    '''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)'''
+    if not core_inst is None:
+        config = core_inst.config
+
+    if config.get('general.random_bind_ip', True):
+        hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
+        data = '.'.join(hostOctets)
+        # Try to bind IP. Some platforms like Mac block non normal 127.x.x.x
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            s.bind((data, 0))
+        except OSError:
+            # if mac/non-bindable, show warning and default to 127.0.0.1
+            logger.warn('Your platform appears to not support random local host addresses 127.x.x.x. Falling back to 127.0.0.1.')
+            data = '127.0.0.1'
+        s.close()
+    else:
+        data = '127.0.0.1'
+    if filePath != '':
+        with open(filePath, 'w') as bindFile:
+            bindFile.write(data)
+    return data

onionr/httpapi/fdsafehandler.py

@@ -0,0 +1,21 @@
+from gevent.pywsgi import WSGIServer, WSGIHandler
+from gevent import Timeout
+class FDSafeHandler(WSGIHandler):
+    '''Our WSGI handler. Doesn't do much non-default except timeouts'''
+    def handle(self):
+        self.timeout = Timeout(120, Exception)
+        self.timeout.start()
+        try:
+            WSGIHandler.handle(self)
+        except Exception:
+            self.handle_error()
+        finally:
+            self.timeout.close()
+
+    def handle_error(self):
+        if v is self.timeout:
+            self.result = [b"Timeout"]
+            self.start_response("200 OK", [])
+            self.process_result()
+        else:
+            WSGIHandler.handle_error(self) 

onionr/httpapi/miscpublicapi/__init__.py

@@ -1,23 +1,4 @@
-'''
+from . import announce, upload, getblocks, endpoints
-    Onionr - Private P2P Communication
-
-    Public endpoints to do various block sync actions and announcement
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-from . import announce, upload, getblocks
 
 announce = announce.handle_announce # endpoint handler for accepting peer announcements
 upload = upload.accept_upload # endpoint handler for accepting public uploads

onionr/httpapi/miscpublicapi/endpoints.py

@@ -0,0 +1,77 @@
+'''
+    Onionr - Private P2P Communication
+
+    Misc public API endpoints too small to need their own file and that need access to the public api inst
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from flask import Response, Blueprint, request, send_from_directory, abort
+from . import getblocks, upload, announce
+class PublicEndpoints:
+    def __init__(self, public_api):
+        client_API = public_api.clientAPI
+        config = client_API._core.config
+
+        public_endpoints_bp = Blueprint('publicendpoints', __name__)
+        self.public_endpoints_bp = public_endpoints_bp
+        @public_endpoints_bp.route('/')
+        def banner():
+            # Display a bit of information to people who visit a node address in their browser
+            try:
+                with open('static-data/index.html', 'r') as html:
+                    resp = Response(html.read(), mimetype='text/html')
+            except FileNotFoundError:
+                resp = Response("")
+            return resp
+
+        @public_endpoints_bp.route('/getblocklist')
+        def get_block_list():
+            return getblocks.public_block_list(client_API, public_api, request)
+
+        @public_endpoints_bp.route('/getdata/<name>')
+        def get_block_data(name):
+            # Share data for a block if we have it
+            return getblocks.public_get_block_data(client_API, public_api, name)
+
+        @public_endpoints_bp.route('/www/<path:path>')
+        def www_public(path):
+            # A way to share files directly over your .onion
+            if not config.get("www.public.run", True):
+                abort(403)
+            return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
+
+        @public_endpoints_bp.route('/ping')
+        def ping():
+            # Endpoint to test if nodes are up
+            return Response("pong!")
+
+        @public_endpoints_bp.route('/pex')
+        def peer_exchange():
+            response = ','.join(client_API._core.listAdders(recent=3600))
+            if len(response) == 0:
+                response = ''
+            return Response(response)
+
+        @public_endpoints_bp.route('/announce', methods=['post'])
+        def accept_announce():
+            resp = httpapi.miscpublicapi.announce(client_API, request)
+            return resp
+
+        @public_endpoints_bp.route('/upload', methods=['post'])
+        def upload():
+            '''Accept file uploads. In the future this will be done more often than on creation 
+            to speed up block sync
+            '''
+            return upload.accept_upload(client_API, request)

onionr/httpapi/security/client.py

@@ -1,6 +1,25 @@
+'''
+    Onionr - Private P2P Communication
+
+    Process incoming requests to the client api server to validate they are legitimate
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
 import hmac
 from flask import Blueprint, request, abort
-
+from onionrservices import httpheaders
 # Be extremely mindful of this. These are endpoints available without a password
 whitelist_endpoints = ('siteapi.site', 'www', 'staticfiles.onionrhome', 'staticfiles.homedata', 
 'staticfiles.board', 'staticfiles.profiles', 
@@ -9,24 +28,30 @@ whitelist_endpoints = ('siteapi.site', 'www', 'staticfiles.onionrhome', 'staticf
 'staticfiles.mail', 'staticfiles.mailindex', 'staticfiles.friends', 'staticfiles.friendsindex',
 'staticfiles.clandestine', 'staticfiles.clandestineIndex')
 
-@app.before_request
+class ClientAPISecurity:
-def validateRequest():
+    def __init__(self, client_api):
+        client_api_security_bp = Blueprint('clientapisecurity', __name__)
+        self.client_api_security_bp = client_api_security_bp
+        self.client_api = client_api
+
+        @client_api_security_bp.before_app_request
+        def validate_request():
             '''Validate request has set password and is the correct hostname'''
             # For the purpose of preventing DNS rebinding attacks
-    if request.host != '%s:%s' % (self.host, self.bindPort):
+            if request.host != '%s:%s' % (client_api.host, client_api.bindPort):
                 abort(403)
             if request.endpoint in whitelist_endpoints:
                 return
             try:
-        if not hmac.compare_digest(request.headers['token'], self.clientToken):
+                if not hmac.compare_digest(request.headers['token'], client_api.clientToken):
-            if not hmac.compare_digest(request.form['token'], self.clientToken):
+                    if not hmac.compare_digest(request.form['token'], client_api.clientToken):
                         abort(403)
             except KeyError:
-        if not hmac.compare_digest(request.form['token'], self.clientToken):
+                if not hmac.compare_digest(request.form['token'], client_api.clientToken):
                     abort(403)
 
-@app.after_request
+        @client_api_security_bp.after_app_request
-def afterReq(resp):
+        def after_req(resp):
             # Security headers
             resp = httpheaders.set_default_onionr_http_headers(resp)
             if request.endpoint == 'site':

onionr/httpapi/security/public.py

@@ -0,0 +1,50 @@
+'''
+    Onionr - Private P2P Communication
+
+    Process incoming requests to the public api server for certain attacks
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from flask import Blueprint, request, abort
+from onionrservices import httpheaders
+from onionrutils import epoch
+
+class PublicAPISecurity:
+    def __init__(self, public_api):
+        public_api_security_bp = Blueprint('publicapisecurity', __name__)
+        self.public_api_security_bp = public_api_security_bp
+
+        @public_api_security_bp.before_app_request
+        def validate_request():
+            '''Validate request has the correct hostname'''
+            # If high security level, deny requests to public (HS should be disabled anyway for Tor, but might not be for I2P)
+            if public_api.config.get('general.security_level', default=1) > 0:
+                abort(403)
+            if type(public_api.torAdder) is None and type(public_api.i2pAdder) is None:
+                # abort if our hs addresses are not known
+                abort(403)
+            if request.host not in (public_api.i2pAdder, public_api.torAdder):
+                # Disallow connection if wrong HTTP hostname, in order to prevent DNS rebinding attacks
+                abort(403)
+            public_api.hitCount += 1 # raise hit count for valid requests
+
+        @public_api_security_bp.after_app_request
+        def send_headers(resp):
+            '''Send api, access control headers'''
+            resp = httpheaders.set_default_onionr_http_headers(resp)
+            # Network API version
+            resp.headers['X-API'] = public_api.API_VERSION
+            public_api.lastRequest = epoch.get_rounded_epoch(roundS=5)
+            return resp

onionr/onionrservices/connectionserver.py

@@ -23,7 +23,7 @@ from flask import Flask
 import core, logger, httpapi
 import onionrexceptions
 from netcontroller import getOpenPort
-import api
+from httpapi import apiutils
 from onionrutils import stringvalidators, basicrequests, bytesconverter
 from . import httpheaders
 
@@ -40,7 +40,7 @@ class ConnectionServer:
         socks = core_inst.config.get('tor.socksport') # Load config for Tor socks port for proxy
         service_app = Flask(__name__) # Setup Flask app for server.
         service_port = getOpenPort()
-        service_ip = api.setBindIP()
+        service_ip = apiutils.setbindip.set_bind_IP(core_inst=self.core_inst)
         http_server = WSGIServer(('127.0.0.1', service_port), service_app, log=None)
         core_inst.onionrInst.communicatorInst.service_greenlets.append(http_server)