renamed onionr dir and bugfixes/linting progress

src/onionrservices/README.md

@@ -0,0 +1,13 @@
+# onionrservices
+
+onionservices is a submodule to handle direct connections to Onionr peers, using the Onionr network to broker them.
+
+## Files
+
+__init__.py: Contains the OnionrServices class which can create direct connection servers or clients.
+
+bootstrapservice.py: Creates a bootstrap server for a peer and announces the connection by creating a block encrypted to the peer we want to connect to.
+
+connectionserver.py: Creates a direct connection server for a peer
+
+httpheaders.py: Modifies a Flask response object http response headers for security purposes.

src/onionrservices/__init__.py

@@ -0,0 +1,64 @@
+'''
+    Onionr - Private P2P Communication
+
+    Onionr services provide the server component to direct connections
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import time
+import stem
+from . import connectionserver, bootstrapservice, serverexists
+from onionrutils import stringvalidators, basicrequests
+import config
+server_exists = serverexists.server_exists
+class OnionrServices:
+    '''
+        Create a client or server for connecting to peer interfaces
+    '''
+    def __init__(self):
+        self.servers = {}
+        self.clients = {}
+        self.shutdown = False
+    
+    def create_server(self, peer, address, comm_inst):
+        '''
+            When a client wants to connect, contact their bootstrap address and tell them our
+            ephemeral address for our service by creating a new ConnectionServer instance
+        '''
+        if not stringvalidators.validate_transport(address): raise ValueError('address must be valid')
+        BOOTSTRAP_TRIES = 10 # How many times to attempt contacting the bootstrap server
+        TRY_WAIT = 3 # Seconds to wait before trying bootstrap again
+        # HTTP is fine because .onion/i2p is encrypted/authenticated
+        base_url = 'http://%s/' % (address,)
+        socks = config.get('tor.socksport')
+        for x in range(BOOTSTRAP_TRIES):
+            if basicrequests.do_get_request(base_url + 'ping', port=socks, ignoreAPI=True) == 'pong!':
+                # if bootstrap sever is online, tell them our service address
+                connectionserver.ConnectionServer(peer, address, comm_inst=comm_inst)
+            else:
+                time.sleep(TRY_WAIT)
+        else:
+            return False
+
+    @staticmethod
+    def create_client(peer, comm_inst=None):
+        # Create ephemeral onion service to bootstrap connection to server
+        if not comm_inst == None:
+            try:
+                return comm_inst.direct_connection_clients[peer]
+            except KeyError:
+                pass
+        address = bootstrapservice.bootstrap_client_service(peer, comm_inst)
+        return address

src/onionrservices/bootstrapservice.py

@@ -0,0 +1,116 @@
+'''
+    Onionr - Private P2P Communication
+
+    Bootstrap onion direct connections for the clients
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import time, threading, uuid, os
+from gevent.pywsgi import WSGIServer, WSGIHandler
+from stem.control import Controller
+from flask import Flask, Response
+from netcontroller import get_open_port
+from . import httpheaders
+from onionrutils import stringvalidators, epoch
+import logger
+import config, onionrblocks, filepaths
+import onionrexceptions
+import deadsimplekv as simplekv
+from . import pool
+
+def __bootstrap_timeout(server: WSGIServer, timeout: int, signal_object):
+    time.sleep(timeout)
+    signal_object.timed_out = True
+    server.stop()
+
+def bootstrap_client_service(peer, comm_inst=None, bootstrap_timeout=300):
+    '''
+        Bootstrap client services
+    '''
+    if not stringvalidators.validate_pub_key(peer):
+        raise ValueError('Peer must be valid base32 ed25519 public key')
+    
+    connection_pool = None
+
+    # here we use a lambda for the timeout thread to set to true
+    timed_out = lambda: None
+    timed_out.timed_out = False
+
+    bootstrap_port = get_open_port()
+    bootstrap_app = Flask(__name__)
+    bootstrap_app.config['MAX_CONTENT_LENGTH'] = 1 * 1024
+
+    http_server = WSGIServer(('127.0.0.1', bootstrap_port), bootstrap_app, log=None)
+    try:
+        if comm_inst is None: raise ValueError
+    except (AttributeError, ValueError) as e:
+        pass
+    else:
+        comm_inst.service_greenlets.append(http_server)
+        connection_pool = comm_inst.shared_state.get(pool.ServicePool)
+    
+    bootstrap_address = ''
+    shutdown = False
+    bs_id = str(uuid.uuid4())
+    key_store = simplekv.DeadSimpleKV(filepaths.cached_storage)
+
+    @bootstrap_app.route('/ping')
+    def get_ping():
+        return "pong!"
+
+    @bootstrap_app.after_request
+    def afterReq(resp):
+        # Security headers
+        resp = httpheaders.set_default_onionr_http_headers(resp)
+        return resp
+
+    @bootstrap_app.route('/bs/<address>', methods=['POST'])
+    def get_bootstrap(address):
+        if stringvalidators.validate_transport(address + '.onion'):
+            # Set the bootstrap address then close the server
+            bootstrap_address = address + '.onion'
+            key_store.put(bs_id, bootstrap_address)
+            http_server.stop()
+            return Response("success")
+        else:
+            return Response("")
+
+    with Controller.from_port(port=config.get('tor.controlPort')) as controller:
+        if not connection_pool is None: connection_pool.bootstrap_pending.append(peer)
+        # Connect to the Tor process for Onionr
+        controller.authenticate(config.get('tor.controlpassword'))
+        # Create the v3 onion service
+        response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, key_type = 'NEW', key_content = 'ED25519-V3', await_publication = True)
+        onionrblocks.insert(response.service_id, header='con', sign=True, encryptType='asym', 
+        asymPeer=peer, disableForward=True, expire=(epoch.get_epoch() + bootstrap_timeout))
+        
+        threading.Thread(target=__bootstrap_timeout, args=[http_server, bootstrap_timeout, timed_out], daemon=True).start()
+
+        # Run the bootstrap server
+        try:
+            http_server.serve_forever()
+        except TypeError:
+            pass
+        # This line reached when server is shutdown by being bootstrapped
+    # Add the address to the client pool
+    if not comm_inst is None:
+        connection_pool.bootstrap_pending.remove(peer)
+        if timed_out.timed_out:
+            logger.warn('Could not connect to %s due to timeout' % (peer,))
+            return None
+        comm_inst.direct_connection_clients[peer] = response.service_id
+
+    # Now that the bootstrap server has received a server, return the address
+    return key_store.get(bs_id)

src/onionrservices/connectionserver.py

@@ -0,0 +1,89 @@
+'''
+    Onionr - Private P2P Communication
+
+    This module does the second part of the bootstrap block handshake and creates the API server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from gevent.pywsgi import WSGIServer
+from stem.control import Controller
+from flask import Flask
+import logger, httpapi
+import onionrexceptions, config, filepaths
+from netcontroller import get_open_port
+from httpapi import apiutils
+from onionrutils import stringvalidators, basicrequests, bytesconverter
+from . import httpheaders
+import deadsimplekv as simplekv
+
+class ConnectionServer:
+    def __init__(self, peer, address, comm_inst=None):
+
+        if not stringvalidators.validate_pub_key(peer):
+            raise ValueError('Peer must be valid base32 ed25519 public key')
+        
+        socks = config.get('tor.socksport') # Load config for Tor socks port for proxy
+        service_app = Flask(__name__) # Setup Flask app for server.
+        service_port = get_open_port()
+        service_ip = apiutils.setbindip.set_bind_IP()
+        http_server = WSGIServer(('127.0.0.1', service_port), service_app, log=None)
+        comm_inst.service_greenlets.append(http_server)
+        key_store = simplekv.DeadSimpleKV(filepaths.cached_storage)
+
+        # TODO define basic endpoints useful for direct connections like stats
+        
+        httpapi.load_plugin_blueprints(service_app, blueprint='direct_blueprint')
+
+        @service_app.route('/ping')
+        def get_ping():
+            return "pong!"
+        
+        @service_app.route('/close')
+        def shutdown_server():
+            comm_inst.service_greenlets.remove(http_server)
+            http_server.stop()
+            return Response('goodbye')
+
+        @service_app.after_request
+        def afterReq(resp):
+            # Security headers
+            resp = httpheaders.set_default_onionr_http_headers(resp)
+            return resp
+
+        with Controller.from_port(port=config.get('tor.controlPort')) as controller:
+            # Connect to the Tor process for Onionr
+            controller.authenticate(config.get('tor.controlpassword'))
+            # Create the v3 onion service for the peer to connect to
+            response = controller.create_ephemeral_hidden_service({80: service_port}, await_publication = True, key_type='NEW', key_content = 'ED25519-V3')
+
+            try:
+                for x in range(3):
+                    attempt = basicrequests.do_post_request('http://' + address + '/bs/' + response.service_id, port=socks)
+                    if attempt == 'success':
+                        break
+                else:
+                    raise ConnectionError
+            except ConnectionError:
+                # Re-raise
+                raise ConnectionError('Could not reach %s bootstrap address %s' % (peer, address))
+            else:
+                # If no connection error, create the service and save it to local global key store
+                peer = bytesconverter.bytes_to_str(peer)
+                key_store.put('dc-' + peer, response.service_id)
+                key_store.flush()
+                logger.info('hosting on %s with %s' % (response.service_id,  peer))
+                http_server.serve_forever()
+                http_server.stop()
+                key_store.delete('dc-' + peer)

src/onionrservices/httpheaders.py

@@ -0,0 +1,35 @@
+'''
+    Onionr - Private P2P Communication
+
+    Set default onionr http headers
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+FEATURE_POLICY = """vibrate; vr; webauthn; usb; sync-xhr; speaker; 
+picture-in-picture; payment; midi; microphone; magnetometer; gyroscope; 
+geolocation; fullscreen; encrypted-media; document-domain; 
+camera; accelerometer; ambient-light-sensor""".replace('\n', '') # have to remove \n for flask
+def set_default_onionr_http_headers(flask_response):
+    '''Response headers'''
+    flask_response.headers['Content-Security-Policy'] = "default-src 'none'; style-src data: 'unsafe-inline'; img-src data:"
+    flask_response.headers['X-Frame-Options'] = 'deny'
+    flask_response.headers['X-Content-Type-Options'] = "nosniff"
+    flask_response.headers['Server'] = ''
+    flask_response.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch.
+    flask_response.headers['Connection'] = "close"
+    flask_response.headers['Clear-Site-Data'] = '"cache", "cookies", "storage", "executionContexts"'
+    flask_response.headers['Feature-Policy'] = FEATURE_POLICY
+    flask_response.headers['Referrer-Policy'] = 'same-origin'
+    return flask_response

src/onionrservices/pool.py

@@ -0,0 +1,29 @@
+'''
+    Onionr - Private P2P Communication
+
+    Holds active onionrservices clients and servers
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from onionrutils import epoch
+class ServicePool:
+    def __init__(self):
+        self.servers = []
+        self.clients = []
+        self.bootstrap_pending = []
+    
+    def add_server(self, service):
+        self.servers.append((service, epoch.get_epoch()))
+        

src/onionrservices/serverexists.py

@@ -0,0 +1,30 @@
+'''
+    Onionr - Private P2P Communication
+
+    Function to check if an onion server is created for a peer or not
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import deadsimplekv
+
+import filepaths
+from onionrutils import bytesconverter
+
+def server_exists(peer: str) -> bool:
+    '''checks if an onion server is created for a peer or not'''
+    peer = bytesconverter.bytes_to_str(peer)
+    kv = deadsimplekv.DeadSimpleKV(filepaths.cached_storage)
+    kv.refresh()
+    return not kv.get('dc-' + peer) is None

src/onionrservices/warden/__init__.py

@@ -0,0 +1,3 @@
+from . import client # Client connection warden. Monitors & validates connection security.
+from . import server # Server connection warden. Monitors and validates server security
+#from . import watchdog # Watchdog. Oversees running services for statistic collection and TTL control

src/onionrservices/warden/bootstrap.py

@@ -0,0 +1,19 @@
+'''
+    Onionr - Private P2P Communication
+
+    Bootstrap warden monitors the bootstrap server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''