renamed onionr dir and bugfixes/linting progress

src/__init__.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+'''
+    Onionr - Private P2P Communication
+
+    This file initializes Onionr when ran to be a daemon or with commands
+
+    Run with 'help' for usage.
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+# Set the user's locale for encoding reasons
+import locale # noqa
+locale.setlocale(locale.LC_ALL, '')
+
+ran_as_script = False
+if __name__ == "__main__": ran_as_script = True
+
+# Import standard libraries
+import sys # noqa
+
+try:
+    from etc import dependencycheck # noqa
+except ModuleNotFoundError as e:
+    print('Onionr needs ' + str(e) + ' installed')
+
+# Onionr imports
+from etc import onionrvalues # For different Onionr related constants such as versions
+import onionrsetup as setup
+
+min_ver = onionrvalues.MIN_PY_VERSION
+
+# Ensure we have at least the minimum python version
+if sys.version_info[0] == 2 or sys.version_info[1] < min_ver:
+    sys.stderr.write('Error, Onionr requires Python 3.' + str(min_ver) + '\n')
+    sys.exit(1)
+
+# Create Onionr data directories, must be done before most imports
+from utils import createdirs
+createdirs.create_dirs()
+
+from onionrcommands import parser
+from onionrplugins import onionrevents as events
+
+setup.setup_config()
+setup.setup_default_plugins()
+
+
+def onionr_main():
+    """Onionr entrypoint, start command processor"""
+    parser.register()
+
+
+if ran_as_script:
+    onionr_main()
+
+    # Cleanup standard out/err because Python refuses to do it itsself
+    try:
+        sys.stderr.close()
+    except (IOError, BrokenPipeError):
+        pass
+    try:
+        sys.stdout.close()
+    except (IOError, BrokenPipeError):
+        pass

src/apiservers/README.md

@@ -0,0 +1,9 @@
+# API Servers
+
+Contains the WSGI servers Onionr uses for remote peer communication and local daemon control
+
+## Files
+
+* \_\_init\_\_.py: Exposes the server classes
+* private: Contains the client API (the server used to interact with the local Onionr daemon, and view the web UI)
+* public: Contains the public API (the server used by remote peers to talk to our daemon)

src/apiservers/__init__.py

@@ -0,0 +1,3 @@
+from . import public, private
+PublicAPI = public.PublicAPI
+ClientAPI = private.PrivateAPI

src/apiservers/private/README.md

@@ -0,0 +1,8 @@
+# Private API Server
+
+Private API server, used to access the web interface locally and control Onionr
+
+## Files
+
+* \_\_init\_\_.py: Sets up the server and a few misc functions
+* register_private_blueprints.py: Adds in flask blueprints for various sub-APIs

src/apiservers/private/__init__.py

@@ -0,0 +1,95 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file handles all incoming http requests to the client, using Flask
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import base64, os, time
+import flask
+from gevent.pywsgi import WSGIServer
+from onionrutils import epoch
+import httpapi, filepaths, logger
+from . import register_private_blueprints
+from etc import waitforsetvar
+import serializeddata, config
+from .. import public
+class PrivateAPI:
+    '''
+        Client HTTP api
+    '''
+
+    callbacks = {'public' : {}, 'private' : {}}
+
+    def __init__(self):
+        '''
+            Initialize the api server, preping variables for later use
+
+            This initialization defines all of the API entry points and handlers for the endpoints and errors
+            This also saves the used host (random localhost IP address) to the data folder in host.txt
+        '''
+        self.config = config
+        self.startTime = epoch.get_epoch()
+        app = flask.Flask(__name__)
+        bindPort = int(config.get('client.client.port', 59496))
+        self.bindPort = bindPort
+
+        self.clientToken = config.get('client.webpassword')
+        self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
+
+        self.host = httpapi.apiutils.setbindip.set_bind_IP(filepaths.private_API_host_file)
+        logger.info('Running api on %s:%s' % (self.host, self.bindPort))
+        self.httpServer = ''
+
+        self.queueResponse = {}
+        self.get_block_data = httpapi.apiutils.GetBlockData(self)
+        register_private_blueprints.register_private_blueprints(self, app)
+        httpapi.load_plugin_blueprints(app)
+        self.app = app
+    
+    def start(self):
+        waitforsetvar.wait_for_set_var(self, "_too_many")
+        self.publicAPI = self._too_many.get(public.PublicAPI)
+        self.httpServer = WSGIServer((self.host, self.bindPort), self.app, log=None, handler_class=httpapi.fdsafehandler.FDSafeHandler)
+        self.httpServer.serve_forever()
+
+    def setPublicAPIInstance(self, inst):
+        self.publicAPI = inst
+
+    def validateToken(self, token):
+        '''
+            Validate that the client token matches the given token. Used to prevent CSRF and data exfiltration
+        '''
+        if not self.clientToken:
+            logger.error("client password needs to be set")
+            return False
+        try:
+            if not hmac.compare_digest(self.clientToken, token):
+                return False
+            else:
+                return True
+        except TypeError:
+            return False
+
+    def getUptime(self)->int:
+        while True:
+            try:
+                return epoch.get_epoch() - self.startTime
+            except (AttributeError, NameError):
+                # Don't error on race condition with startup
+                pass
+
+    def getBlockData(self, bHash, decrypt=False, raw=False, headerOnly=False):
+        return self.get_block_data.get_block_data(bHash, decrypt=decrypt, raw=raw, headerOnly=headerOnly)

src/apiservers/private/register_private_blueprints.py

@@ -0,0 +1,39 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file registers blueprints for the private api server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import os
+from httpapi import security, friendsapi, profilesapi, configapi, insertblock, miscclientapi, onionrsitesapi, apiutils
+from httpapi import directconnections
+from httpapi import themeapi
+
+def register_private_blueprints(private_api, app):
+    app.register_blueprint(security.client.ClientAPISecurity(private_api).client_api_security_bp)
+    app.register_blueprint(friendsapi.friends)
+    app.register_blueprint(profilesapi.profile_BP)
+    app.register_blueprint(configapi.config_BP)
+    app.register_blueprint(insertblock.ib)
+    app.register_blueprint(miscclientapi.getblocks.client_get_blocks)
+    app.register_blueprint(miscclientapi.endpoints.PrivateEndpoints(private_api).private_endpoints_bp)
+    app.register_blueprint(miscclientapi.motd.bp)
+    app.register_blueprint(onionrsitesapi.site_api)
+    app.register_blueprint(apiutils.shutdown.shutdown_bp)
+    app.register_blueprint(miscclientapi.staticfiles.static_files_bp)
+    app.register_blueprint(directconnections.DirectConnectionManagement(private_api).direct_conn_management_bp)
+    app.register_blueprint(themeapi.theme_blueprint)
+    return app

src/apiservers/public/__init__.py

@@ -0,0 +1,64 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file handles all incoming http requests to the public api server, using Flask
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import time
+import threading
+import flask
+from gevent.pywsgi import WSGIServer
+from httpapi import apiutils, security, fdsafehandler, miscpublicapi
+import logger, config, filepaths
+from utils import gettransports
+from etc import onionrvalues, waitforsetvar
+
+def _get_tor_adder(pub_api):
+    transports = []
+    while len(transports) == 0:
+        transports = gettransports.get()
+        time.sleep(0.3)
+    pub_api.torAdder = transports[0]
+
+class PublicAPI:
+    '''
+        The new client api server, isolated from the public api
+    '''
+    def __init__(self):
+        app = flask.Flask('PublicAPI')
+        app.config['MAX_CONTENT_LENGTH'] = 5 * 1024 * 1024
+        self.i2pEnabled = config.get('i2p.host', False)
+        self.hideBlocks = [] # Blocks to be denied sharing
+        self.host = apiutils.setbindip.set_bind_IP(filepaths.public_API_host_file)
+
+        threading.Thread(target=_get_tor_adder, args=[self], daemon=True).start()
+
+        self.torAdder = ""
+        self.bindPort = config.get('client.public.port')
+        self.lastRequest = 0
+        self.hitCount = 0 # total rec requests to public api since server started
+        self.config = config
+        self.API_VERSION = onionrvalues.API_VERSION
+        logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
+
+        app.register_blueprint(security.public.PublicAPISecurity(self).public_api_security_bp)
+        app.register_blueprint(miscpublicapi.endpoints.PublicEndpoints(self).public_endpoints_bp)
+        self.app = app
+
+    def start(self):
+        waitforsetvar.wait_for_set_var(self, "_too_many")
+        self.httpServer = WSGIServer((self.host, self.bindPort), self.app, log=None, handler_class=fdsafehandler.FDSafeHandler)
+        self.httpServer.serve_forever()

src/communicator/README.md

@@ -0,0 +1,15 @@
+# Onionr Communicator
+
+Onionr communicator is the Onionr client. It "connects" to remote Onionr peers and does things such as:
+
+* Finding new peers
+* Uploading blocks
+* Downloading blocks
+* Daemon maintenance/housekeeping
+
+## Files
+
+* \_\_init\_\_.py: Contains the main communicator code. Inits and launches the communicator and sets up the timers
+* peeraction.py: contains a function to send commands to remote peers
+* bootstrappers.py: adds peers from the bootstrap list to the communicator to try to connect to them
+* onlinepeers: management of the online peer pool for the communicator

src/communicator/__init__.py

@@ -0,0 +1,270 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file contains both the OnionrCommunicate class for communcating with peers
+    and code to operate as a daemon, getting commands from the command queue database (see core.Core.daemonQueue)
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys, os, time
+
+import config, logger
+import onionrexceptions, onionrpeers
+from onionrblocks import onionrblockapi as block
+from onionrplugins import onionrevents as events
+import onionrplugins as plugins
+from . import onlinepeers, uploadqueue
+from communicatorutils import servicecreator, onionrcommunicatortimers
+from communicatorutils import downloadblocks, lookupblocks, lookupadders
+from communicatorutils import servicecreator, connectnewpeers
+from communicatorutils import uploadblocks
+from communicatorutils import daemonqueuehandler, announcenode, deniableinserts
+from communicatorutils import cooldownpeer, housekeeping, netcheck
+from onionrutils import localcommand, epoch
+from etc import humanreadabletime
+import onionrservices, filepaths
+from onionrblocks import storagecounter
+from coredb import daemonqueue, dbfiles
+from utils import gettransports
+from netcontroller import NetController
+OnionrCommunicatorTimers = onionrcommunicatortimers.OnionrCommunicatorTimers
+
+config.reload()
+class OnionrCommunicatorDaemon:
+    def __init__(self, shared_state, developmentMode=config.get('general.dev_mode', False)):
+        # configure logger and stuff
+        self.config = config
+        self.storage_counter = storagecounter.StorageCounter()
+        self.isOnline = True # Assume we're connected to the internet
+        self.shared_state = shared_state # TooManyObjects module
+
+        # list of timer instances
+        self.timers = []
+
+        # initialize core with Tor socks port being 3rd argument
+        self.proxyPort = shared_state.get(NetController).socksPort
+
+        # Upload information, list of blocks to upload
+        self.blocksToUpload = []
+        self.upload_session_manager = self.shared_state.get(uploadblocks.sessionmanager.BlockUploadSessionManager)
+        self.shared_state.share_object()
+
+        # loop time.sleep delay in seconds
+        self.delay = 1
+
+        # lists of connected peers and peers we know we can't reach currently
+        self.onlinePeers = []
+        self.offlinePeers = []
+        self.cooldownPeer = {}
+        self.connectTimes = {}
+        self.peerProfiles = [] # list of peer's profiles (onionrpeers.PeerProfile instances)
+        self.newPeers = [] # Peers merged to us. Don't add to db until we know they're reachable
+        self.announceProgress = {}
+        self.announceCache = {}
+
+        self.generating_blocks = []
+
+        # amount of threads running by name, used to prevent too many
+        self.threadCounts = {}
+
+        # set true when shutdown command received
+        self.shutdown = False
+
+        # list of new blocks to download, added to when new block lists are fetched from peers
+        self.blockQueue = {}
+
+        # list of blocks currently downloading, avoid s
+        self.currentDownloading = []
+
+        # timestamp when the last online node was seen
+        self.lastNodeSeen = None
+
+        # Dict of time stamps for peer's block list lookup times, to avoid downloading full lists all the time
+        self.dbTimestamps = {}
+
+        # Clear the daemon queue for any dead messages
+        if os.path.exists(dbfiles.daemon_queue_db):
+            daemonqueue.clear_daemon_queue()
+
+        # Loads in and starts the enabled plugins
+        plugins.reload()
+
+        # time app started running for info/statistics purposes
+        self.startTime = epoch.get_epoch()
+
+        uploadqueue.UploadQueue(self) # extends our upload list and saves our list when Onionr exits
+
+        if developmentMode:
+            OnionrCommunicatorTimers(self, self.heartbeat, 30)
+
+        # Set timers, function reference, seconds
+        # requires_peer True means the timer function won't fire if we have no connected peers
+        peerPoolTimer = OnionrCommunicatorTimers(self, onlinepeers.get_online_peers, 60, max_threads=1, my_args=[self])
+        OnionrCommunicatorTimers(self, self.runCheck, 2, max_threads=1)
+
+        # Timers to periodically lookup new blocks and download them
+        lookup_blocks_timer = OnionrCommunicatorTimers(self, lookupblocks.lookup_blocks_from_communicator, config.get('timers.lookupBlocks', 25), my_args=[self], requires_peer=True, max_threads=1)
+        # The block download timer is accessed by the block lookup function to trigger faster download starts
+        self.download_blocks_timer = OnionrCommunicatorTimers(self, self.getBlocks, config.get('timers.getBlocks', 10), requires_peer=True, max_threads=5)
+
+        # Timer to reset the longest offline peer so contact can be attempted again
+        OnionrCommunicatorTimers(self, onlinepeers.clear_offline_peer, 58, my_args=[self])
+
+        # Timer to cleanup old blocks
+        blockCleanupTimer = OnionrCommunicatorTimers(self, housekeeping.clean_old_blocks, 20, my_args=[self])
+
+        # Timer to discover new peers
+        OnionrCommunicatorTimers(self, lookupadders.lookup_new_peer_transports_with_communicator, 60, requires_peer=True, my_args=[self], max_threads=2)
+
+        # Timer for adjusting which peers we actively communicate to at any given time, to avoid over-using peers
+        OnionrCommunicatorTimers(self, cooldownpeer.cooldown_peer, 30, my_args=[self], requires_peer=True)
+
+        # Timer to read the upload queue and upload the entries to peers
+        OnionrCommunicatorTimers(self, uploadblocks.upload_blocks_from_communicator, 5, my_args=[self], requires_peer=True, max_threads=1)
+
+        # Timer to process the daemon command queue
+        OnionrCommunicatorTimers(self, daemonqueuehandler.handle_daemon_commands, 6, my_args=[self], max_threads=3)
+
+        # Setup direct connections
+        if config.get('general.socket_servers', False):
+            self.services = onionrservices.OnionrServices()
+            self.active_services = []
+            self.service_greenlets = []
+            OnionrCommunicatorTimers(self, servicecreator.service_creator, 5, max_threads=50, my_args=[self])
+        else:
+            self.services = None
+        
+        # {peer_pubkey: ephemeral_address}, the address to reach them
+        self.direct_connection_clients = {}
+        
+        # This timer creates deniable blocks, in an attempt to further obfuscate block insertion metadata
+        if config.get('general.insert_deniable_blocks', True):
+            deniableBlockTimer = OnionrCommunicatorTimers(self, deniableinserts.insert_deniable_block, 180, my_args=[self], requires_peer=True, max_threads=1)
+            deniableBlockTimer.count = (deniableBlockTimer.frequency - 175)
+
+        # Timer to check for connectivity, through Tor to various high-profile onion services
+        netCheckTimer = OnionrCommunicatorTimers(self, netcheck.net_check, 500, my_args=[self], max_threads=1)
+
+        # Announce the public API server transport address to other nodes if security level allows
+        if config.get('general.security_level', 1) == 0 and config.get('general.announce_node', True):
+            # Default to high security level incase config breaks
+            announceTimer = OnionrCommunicatorTimers(self, announcenode.announce_node, 3600, my_args=[self], requires_peer=True, max_threads=1)
+            announceTimer.count = (announceTimer.frequency - 120)
+        else:
+            logger.debug('Will not announce node.')
+        
+        # Timer to delete malfunctioning or long-dead peers
+        cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requires_peer=True)
+
+        # Timer to cleanup dead ephemeral forward secrecy keys 
+        forwardSecrecyTimer = OnionrCommunicatorTimers(self, housekeeping.clean_keys, 15, my_args=[self], max_threads=1)
+
+        # Adjust initial timer triggers
+        peerPoolTimer.count = (peerPoolTimer.frequency - 1)
+        cleanupTimer.count = (cleanupTimer.frequency - 60)
+        blockCleanupTimer.count = (blockCleanupTimer.frequency - 2)
+        lookup_blocks_timer = (lookup_blocks_timer.frequency - 2)
+
+        shared_state.add(self)
+        
+        if config.get('general.use_bootstrap', True):
+            bootstrappeers.add_bootstrap_list_to_peer_list(self, [], db_only=True)
+
+        # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
+        try:
+            while not self.shutdown:
+                for i in self.timers:
+                    if self.shutdown:
+                        break
+                    i.processTimer()
+                time.sleep(self.delay)
+                # Debug to print out used FDs (regular and net)
+                #proc = psutil.Process()
+                #print(proc.open_files(), len(psutil.net_connections()))
+        except KeyboardInterrupt:
+            self.shutdown = True
+            pass
+
+        logger.info('Goodbye. (Onionr is cleaning up, and will exit)', terminal=True)
+        try:
+            self.service_greenlets
+        except AttributeError:
+            pass
+        else:
+            # Stop onionr direct connection services
+            for server in self.service_greenlets:
+                server.stop()
+        localcommand.local_command('shutdown') # shutdown the api
+        try:
+            time.sleep(0.5)
+        except KeyboardInterrupt:
+            pass
+
+    def getBlocks(self):
+        '''download new blocks in queue'''
+        downloadblocks.download_blocks_from_communicator(self)
+
+    def decrementThreadCount(self, threadName):
+        '''Decrement amount of a thread name if more than zero, called when a function meant to be run in a thread ends'''
+        try:
+            if self.threadCounts[threadName] > 0:
+                self.threadCounts[threadName] -= 1
+        except KeyError:
+            pass
+
+    def connectNewPeer(self, peer='', useBootstrap=False):
+        '''Adds a new random online peer to self.onlinePeers'''
+        connectnewpeers.connect_new_peer_to_communicator(self, peer, useBootstrap)
+
+    def peerCleanup(self):
+        '''This just calls onionrpeers.cleanupPeers, which removes dead or bad peers (offline too long, too slow)'''
+        onionrpeers.peer_cleanup()
+        self.decrementThreadCount('peerCleanup')
+
+    def getPeerProfileInstance(self, peer):
+        '''Gets a peer profile instance from the list of profiles, by address name'''
+        for i in self.peerProfiles:
+            # if the peer's profile is already loaded, return that
+            if i.address == peer:
+                retData = i
+                break
+        else:
+            # if the peer's profile is not loaded, return a new one. connectNewPeer also adds it to the list on connect
+            retData = onionrpeers.PeerProfiles(peer)
+            self.peerProfiles.append(retData)
+        return retData
+
+    def getUptime(self):
+        return epoch.get_epoch() - self.startTime
+
+    def heartbeat(self):
+        '''Show a heartbeat debug message'''
+        logger.debug('Heartbeat. Node running for %s.' % humanreadabletime.human_readable_time(self.getUptime()))
+        self.decrementThreadCount('heartbeat')
+
+    def runCheck(self):
+        if run_file_exists(self):
+            logger.debug('Status check; looks good.')
+
+        self.decrementThreadCount('runCheck')
+
+def startCommunicator(shared_state):
+    OnionrCommunicatorDaemon(shared_state)
+
+def run_file_exists(daemon):
+    if os.path.isfile(filepaths.run_check_file):
+        os.remove(filepaths.run_check_file)
+        return True
+    return False

src/communicator/bootstrappeers.py

@@ -0,0 +1,31 @@
+'''
+    Onionr - Private P2P Communication
+
+    add bootstrap peers to the communicator peer list
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from utils import readstatic, gettransports
+from coredb import keydb
+bootstrap_peers = readstatic.read_static('bootstrap-nodes.txt').split(',')
+
+def add_bootstrap_list_to_peer_list(comm_inst, peerList, db_only=False):
+    '''
+        Add the bootstrap list to the peer list (no duplicates)
+    '''
+    for i in bootstrap_peers:
+        if i not in peerList and i not in comm_inst.offlinePeers and not i in gettransports.get() and len(str(i).strip()) > 0:
+            if not db_only: peerList.append(i)
+            keydb.addkeys.add_address(i)

src/communicator/onlinepeers/README.md

@@ -0,0 +1,12 @@
+# Online Peers
+
+Manages a pool of peers to perform actions with. Since Onionr does not maintain socket connections, it holds a list of peers.
+
+
+## Files
+
+* \_\_init\_\_.py: exposes some functions to interact with the pool
+* clearofflinepeer.py: Pop the oldest peer in the offline list
+* onlinepeers.py: communicator timer to add new peers to the pool randomly
+* pickonlinepeers.py: returns a random peer from the online pool
+* removeonlinepeer.py: removes a specified peer from the online pool

src/communicator/onlinepeers/__init__.py

@@ -0,0 +1,6 @@
+from . import clearofflinepeer, onlinepeers, pickonlinepeers, removeonlinepeer
+
+clear_offline_peer = clearofflinepeer.clear_offline_peer
+get_online_peers = onlinepeers.get_online_peers
+pick_online_peer = pickonlinepeers.pick_online_peer
+remove_online_peer = removeonlinepeer.remove_online_peer

src/communicator/onlinepeers/clearofflinepeer.py

@@ -0,0 +1,29 @@
+'''
+    Onionr - Private P2P Communication
+
+    clear offline peer in a communicator instance
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import logger
+def clear_offline_peer(comm_inst):
+    '''Removes the longest offline peer to retry later'''
+    try:
+        removed = comm_inst.offlinePeers.pop(0)
+    except IndexError:
+        pass
+    else:
+        logger.debug('Removed ' + removed + ' from offline list, will try them again.')
+    comm_inst.decrementThreadCount('clear_offline_peer')

src/communicator/onlinepeers/onlinepeers.py

@@ -0,0 +1,45 @@
+'''
+    Onionr - Private P2P Communication
+
+    get online peers in a communicator instance
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import time
+from etc import humanreadabletime
+import logger
+def get_online_peers(comm_inst):
+    '''
+        Manages the comm_inst.onlinePeers attribute list, connects to more peers if we have none connected
+    '''
+    config = comm_inst.config
+    logger.debug('Refreshing peer pool...')
+    maxPeers = int(config.get('peers.max_connect', 10))
+    needed = maxPeers - len(comm_inst.onlinePeers)
+
+    for i in range(needed):
+        if len(comm_inst.onlinePeers) == 0:
+            comm_inst.connectNewPeer(useBootstrap=True)
+        else:
+            comm_inst.connectNewPeer()
+
+        if comm_inst.shutdown:
+            break
+    else:
+        if len(comm_inst.onlinePeers) == 0:
+            logger.debug('Couldn\'t connect to any peers.' + (' Last node seen %s ago.' % humanreadabletime.human_readable_time(time.time() - comm_inst.lastNodeSeen) if not comm_inst.lastNodeSeen is None else ''), terminal=True)
+        else:
+            comm_inst.lastNodeSeen = time.time()
+    comm_inst.decrementThreadCount('get_online_peers')

src/communicator/onlinepeers/pickonlinepeers.py

@@ -0,0 +1,35 @@
+'''
+    Onionr - Private P2P Communication
+
+    pick online peers in a communicator instance
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import secrets
+def pick_online_peer(comm_inst):
+    '''randomly picks peer from pool without bias (using secrets module)'''
+    ret_data = ''
+    while True:
+        peer_length = len(comm_inst.onlinePeers)
+        if peer_length <= 0:
+            break
+        try:
+            # get a random online peer, securely. May get stuck in loop if network is lost or if all peers in pool magically disconnect at once
+            ret_data = comm_inst.onlinePeers[secrets.randbelow(peer_length)]
+        except IndexError:
+            pass
+        else:
+            break
+    return ret_data

src/communicator/onlinepeers/removeonlinepeer.py

@@ -0,0 +1,33 @@
+'''
+    Onionr - Private P2P Communication
+
+    remove an online peer from the pool in a communicator instance
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+def remove_online_peer(comm_inst, peer):
+    '''Remove an online peer'''
+    try:
+        del comm_inst.connectTimes[peer]
+    except KeyError:
+        pass
+    try:
+        del comm_inst.dbTimestamps[peer]
+    except KeyError:
+        pass
+    try:
+        comm_inst.onlinePeers.remove(peer)
+    except ValueError:
+        pass

src/communicator/peeraction.py

@@ -0,0 +1,55 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file implements logic for performing requests to Onionr peers
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import streamedrequests
+import logger
+from onionrutils import epoch, basicrequests
+from coredb import keydb
+from . import onlinepeers
+
+def peer_action(comm_inst, peer, action, returnHeaders=False, max_resp_size=5242880):
+    '''Perform a get request to a peer'''
+    penalty_score = -10
+    if len(peer) == 0:
+        return False
+    url = 'http://%s/%s' % (peer, action)
+
+    try:
+        ret_data = basicrequests.do_get_request(url, port=comm_inst.proxyPort,
+                                                max_size=max_resp_size)
+    except streamedrequests.exceptions.ResponseLimitReached:
+        logger.warn('Request failed due to max response size being overflowed', terminal=True)
+        ret_data = False
+        penalty_score = -100
+    # if request failed, (error), mark peer offline
+    if ret_data == False: # For some reason "if not" breaks this. Prob has to do with empty string.
+        try:
+            comm_inst.getPeerProfileInstance(peer).addScore(penalty_score)
+            onlinepeers.remove_online_peer(comm_inst, peer)
+            keydb.transportinfo.set_address_info(peer, 'lastConnectAttempt', epoch.get_epoch())
+            if action != 'ping' and not comm_inst.shutdown:
+                logger.warn(f'Lost connection to {peer}', terminal=True)
+                onlinepeers.get_online_peers(comm_inst) # Will only add a new peer to pool if needed
+        except ValueError:
+            pass
+    else:
+        peer_profile = comm_inst.getPeerProfileInstance(peer)
+        peer_profile.update_connect_time()
+        peer_profile.addScore(1)
+    return ret_data # If returnHeaders, returns tuple of data, headers. if not, just data string

src/communicator/uploadqueue/__init__.py

@@ -0,0 +1,60 @@
+"""
+    Onionr - Private P2P Communication
+
+    Class to remember blocks that need to be uploaded and not shared on startup/shutdown
+"""
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import atexit
+import json
+
+import deadsimplekv
+
+import filepaths
+from onionrutils import localcommand
+
+UPLOAD_MEMORY_FILE = filepaths.upload_list
+
+def _add_to_hidden_blocks(cache):
+    for bl in cache:
+        localcommand.local_command('waitforshare/' + bl, post=True)
+
+class UploadQueue:
+    """
+        Saves and loads block upload info from json file
+    """
+
+    def __init__(self, communicator: 'OnionrCommunicatorDaemon'):
+        """Start the UploadQueue object, loading left over uploads into queue 
+        and registering save shutdown function
+        """
+        self.communicator = communicator
+        cache = deadsimplekv.DeadSimpleKV(UPLOAD_MEMORY_FILE)
+        self.store_obj = cache
+        cache: list = cache.get('uploads')
+        if cache == None:
+            cache = []
+        
+        _add_to_hidden_blocks(cache)
+        self.communicator.blocksToUpload.extend(cache)
+
+        atexit.register(self.save)
+
+    def save(self):
+        """Saves to disk on shutdown or if called manually"""
+        bl: list = self.communicator.blocksToUpload
+        self.store_obj.put('uploads', bl)
+        self.store_obj.flush()

src/communicatorutils/README.md

@@ -0,0 +1,33 @@
+# communicatorutils
+
+The files in this submodule handle various subtasks and utilities for the onionr communicator.
+
+## Files:
+
+announcenode.py: Uses a communicator instance to announce our transport address to connected nodes
+
+connectnewpeers.py: takes a communicator instance and has it connect to as many peers as needed, and/or to a new specified peer.
+
+cooldownpeer.py: randomly selects a connected peer in a communicator and disconnects them for the purpose of security and network balancing.
+
+daemonqueuehandler.py: checks for new commands in the daemon queue and processes them accordingly.
+
+deniableinserts.py: insert fake blocks with the communicator for plausible deniability
+
+downloadblocks.py: iterates a communicator instance's block download queue and attempts to download the blocks from online peers
+
+housekeeping.py: cleans old blocks and forward secrecy keys
+
+lookupadders.py: ask connected peers to share their list of peer transport addresses
+
+lookupblocks.py: lookup new blocks from connected peers from the communicator
+
+netcheck.py: check if the node is online based on communicator status and onion server ping results
+
+onionrcommunicataortimers.py: create a timer for a function to be launched on an interval. Control how many possible instances of a timer may be running a function at once and control if the timer should be ran in a thread or not.
+
+proxypicker.py: returns a string name for the appropriate proxy to be used with a particular peer transport address.
+
+servicecreator.py: iterate connection blocks and create new direct connection servers for them.
+
+uploadblocks.py: iterate a communicator's upload queue and upload the blocks to connected peers

src/communicatorutils/announcenode.py

@@ -0,0 +1,86 @@
+'''
+    Onionr - Private P2P Communication
+
+    Use a communicator instance to announce our transport address to connected nodes
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import base64
+import onionrproofs, logger
+from etc import onionrvalues
+from onionrutils import basicrequests, bytesconverter
+from utils import gettransports
+from netcontroller import NetController
+from communicator import onlinepeers
+from coredb import keydb
+def announce_node(daemon):
+    '''Announce our node to our peers'''
+    ret_data = False
+    announce_fail = False
+    
+    # Do not let announceCache get too large
+    if len(daemon.announceCache) >= 10000:
+        daemon.announceCache.popitem()
+
+    if daemon.config.get('general.security_level', 0) == 0:
+        # Announce to random online peers
+        for i in daemon.onlinePeers:
+            if not i in daemon.announceCache and not i in daemon.announceProgress:
+                peer = i
+                break
+        else:
+            peer = onlinepeers.pick_online_peer(daemon)
+
+        for x in range(1):
+            try:
+                ourID = gettransports.get()[0]
+            except IndexError:
+                break
+
+            url = 'http://' + peer + '/announce'
+            data = {'node': ourID}
+
+            combinedNodes = ourID + peer
+            if ourID != 1:
+                existingRand = bytesconverter.bytes_to_str(keydb.transportinfo.get_address_info(peer, 'powValue'))
+                # Reset existingRand if it no longer meets the minimum POW
+                if type(existingRand) is type(None) or not existingRand.endswith('0' * onionrvalues.ANNOUNCE_POW):
+                    existingRand = ''
+
+            if peer in daemon.announceCache:
+                data['random'] = daemon.announceCache[peer]
+            elif len(existingRand) > 0:
+                data['random'] = existingRand
+            else:
+                daemon.announceProgress[peer] = True
+                proof = onionrproofs.DataPOW(combinedNodes, minDifficulty=onionrvalues.ANNOUNCE_POW)
+                del daemon.announceProgress[peer]
+                try:
+                    data['random'] = base64.b64encode(proof.waitForResult()[1])
+                except TypeError:
+                    # Happens when we failed to produce a proof
+                    logger.error("Failed to produce a pow for announcing to " + peer)
+                    announce_fail = True
+                else:
+                    daemon.announceCache[peer] = data['random']
+            if not announce_fail:
+                logger.info('Announcing node to ' + url)
+                if basicrequests.do_post_request(url, data, port=daemon.shared_state.get(NetController).socksPort) == 'Success':
+                    logger.info('Successfully introduced node to ' + peer, terminal=True)
+                    ret_data = True
+                    keydb.transportinfo.set_address_info(peer, 'introduced', 1)
+                    keydb.transportinfo.set_address_info(peer, 'powValue', data['random'])
+    daemon.decrementThreadCount('announce_node')
+    return ret_data

src/communicatorutils/connectnewpeers.py

@@ -0,0 +1,94 @@
+'''
+    Onionr - Private P2P Communication
+
+    Connect a new peer to our communicator instance. Does so randomly if no peer is specified
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import time, sys, secrets
+import onionrexceptions, logger, onionrpeers
+from utils import networkmerger, gettransports
+from onionrutils import stringvalidators, epoch
+from communicator import peeraction, bootstrappeers
+from coredb import keydb
+def connect_new_peer_to_communicator(comm_inst, peer='', useBootstrap=False):
+    config = comm_inst.config
+    retData = False
+    tried = comm_inst.offlinePeers
+    transports = gettransports.get()
+    if peer != '':
+        if stringvalidators.validate_transport(peer):
+            peerList = [peer]
+        else:
+            raise onionrexceptions.InvalidAddress('Will not attempt connection test to invalid address')
+    else:
+        peerList = keydb.listkeys.list_adders()
+
+    mainPeerList = keydb.listkeys.list_adders()
+    peerList = onionrpeers.get_score_sorted_peer_list()
+
+    # If we don't have enough peers connected or random chance, select new peers to try
+    if len(peerList) < 8 or secrets.randbelow(4) == 3:
+        tryingNew = []
+        for x in comm_inst.newPeers:
+            if x not in peerList:
+                peerList.append(x)
+                tryingNew.append(x)
+        for i in tryingNew:
+            comm_inst.newPeers.remove(i)
+    
+    if len(peerList) == 0 or useBootstrap:
+        # Avoid duplicating bootstrap addresses in peerList
+        if config.get('general.use_bootstrap_list', True):
+            bootstrappeers.add_bootstrap_list_to_peer_list(comm_inst, peerList)
+
+    for address in peerList:
+        address = address.strip()
+
+        if not config.get('tor.v3onions') and len(address) == 62:
+            continue
+        # Don't connect to our own address
+        if address in transports:
+            continue
+        # Don't connect to invalid address or if its already been tried/connected, or if its cooled down
+        if len(address) == 0 or address in tried or address in comm_inst.onlinePeers or address in comm_inst.cooldownPeer:
+            continue
+        if comm_inst.shutdown:
+            return
+        # Ping a peer,
+        ret = peeraction.peer_action(comm_inst, address, 'ping')
+        if ret == 'pong!':
+            time.sleep(0.1)
+            if address not in mainPeerList:
+                # Add a peer to our list if it isn't already since it successfully connected
+                networkmerger.mergeAdders(address)
+            if address not in comm_inst.onlinePeers:
+                logger.info('Connected to ' + address, terminal=True)
+                comm_inst.onlinePeers.append(address)
+                comm_inst.connectTimes[address] = epoch.get_epoch()
+            retData = address
+
+            # add peer to profile list if they're not in it
+            for profile in comm_inst.peerProfiles:
+                if profile.address == address:
+                    break
+            else:
+                comm_inst.peerProfiles.append(onionrpeers.PeerProfiles(address))
+            break
+        else:
+            # Mark a peer as tried if they failed to respond to ping
+            tried.append(address)
+            logger.debug('Failed to connect to %s: %s ' % (address, ret))
+    return retData

src/communicatorutils/cooldownpeer.py

@@ -0,0 +1,54 @@
+'''
+    Onionr - Private P2P Communication
+
+    Select a random online peer in a communicator instance and have them "cool down"
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from onionrutils import epoch
+from communicator import onlinepeers
+def cooldown_peer(comm_inst):
+    '''Randomly add an online peer to cooldown, so we can connect a new one'''
+    config = comm_inst.config
+    onlinePeerAmount = len(comm_inst.onlinePeers)
+    minTime = 300
+    cooldownTime = 600
+    toCool = ''
+    tempConnectTimes = dict(comm_inst.connectTimes)
+
+    # Remove peers from cooldown that have been there long enough
+    tempCooldown = dict(comm_inst.cooldownPeer)
+    for peer in tempCooldown:
+        if (epoch.get_epoch() - tempCooldown[peer]) >= cooldownTime:
+            del comm_inst.cooldownPeer[peer]
+
+    # Cool down a peer, if we have max connections alive for long enough
+    if onlinePeerAmount >= config.get('peers.max_connect', 10, save = True):
+        finding = True
+
+        while finding:
+            try:
+                toCool = min(tempConnectTimes, key=tempConnectTimes.get)
+                if (epoch.get_epoch() - tempConnectTimes[toCool]) < minTime:
+                    del tempConnectTimes[toCool]
+                else:
+                    finding = False
+            except ValueError:
+                break
+        else:
+            onlinepeers.remove_online_peer(comm_inst, toCool)
+            comm_inst.cooldownPeer[toCool] = epoch.get_epoch()
+
+    comm_inst.decrementThreadCount('cooldown_peer')

src/communicatorutils/daemonqueuehandler.py

@@ -0,0 +1,73 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Handle daemon queue commands in the communicator
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import logger
+from onionrplugins import onionrevents as events
+from onionrutils import localcommand
+from coredb import daemonqueue
+import filepaths
+from . import restarttor
+def handle_daemon_commands(comm_inst):
+    cmd = daemonqueue.daemon_queue()
+    response = ''
+    if cmd is not False:
+        events.event('daemon_command', data = {'cmd' : cmd})
+        if cmd[0] == 'shutdown':
+            comm_inst.shutdown = True
+        elif cmd[0] == 'runtimeTest':
+            comm_inst.shared_state.get_by_string("OnionrRunTestManager").run_tests()
+        elif cmd[0] == 'remove_from_insert_list':
+            try:
+                comm_inst.generating_blocks.remove(cmd[1])
+            except ValueError:
+                pass
+        elif cmd[0] == 'announceNode':
+            if len(comm_inst.onlinePeers) > 0:
+                comm_inst.announce(cmd[1])
+            else:
+                logger.debug("No nodes connected. Will not introduce node.")
+        elif cmd[0] == 'runCheck': # deprecated
+            logger.debug('Status check; looks good.')
+            open(filepaths.run_check_file + '.runcheck', 'w+').close()
+        elif cmd[0] == 'connectedPeers':
+            response = '\n'.join(list(comm_inst.onlinePeers)).strip()
+            if response == '':
+                response = 'none'
+        elif cmd[0] == 'localCommand':
+            response = localcommand.local_command(cmd[1])
+        elif cmd[0] == 'clearOffline':
+            comm_inst.offlinePeers = []
+        elif cmd[0] == 'restartTor':
+            restarttor.restart(comm_inst)
+            comm_inst.offlinePeers = []
+        elif cmd[0] == 'pex':
+            for i in comm_inst.timers:
+                if i.timerFunction.__name__ == 'lookupAdders':
+                    i.count = (i.frequency - 1)
+        elif cmd[0] == 'uploadBlock':
+            comm_inst.blocksToUpload.append(cmd[1])
+        else:
+            logger.debug('Received daemon queue command unable to be handled: %s' % (cmd[0],))
+
+        if cmd[0] not in ('', None):
+            if response != '':
+                localcommand.local_command('queueResponseAdd/' + cmd[4], post=True, postData={'data': response})
+        response = ''
+
+    comm_inst.decrementThreadCount('handle_daemon_commands')

src/communicatorutils/deniableinserts.py

@@ -0,0 +1,32 @@
+'''
+    Onionr - Private P2P Communication
+
+    Use the communicator to insert fake mail messages
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import secrets
+from etc import onionrvalues
+import onionrblocks
+def insert_deniable_block(comm_inst):
+    '''Insert a fake block in order to make it more difficult to track real blocks'''
+    fakePeer = ''
+    chance = 10
+    if secrets.randbelow(chance) == (chance - 1):
+        # This assumes on the libsodium primitives to have key-privacy
+        fakePeer = onionrvalues.DENIABLE_PEER_ADDRESS
+        data = secrets.token_hex(secrets.randbelow(1024) + 1)
+        onionrblocks.insert(data, header='pm', encryptType='asym', asymPeer=fakePeer, disableForward=True, meta={'subject': 'foo'})
+    comm_inst.decrementThreadCount('insert_deniable_block')

src/communicatorutils/downloadblocks/__init__.py

@@ -0,0 +1,133 @@
+'''
+    Onionr - Private P2P Communication
+
+    Download blocks using the communicator instance
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import communicator, onionrexceptions
+import logger, onionrpeers
+from onionrutils import blockmetadata, stringvalidators, validatemetadata
+from coredb import blockmetadb
+from . import shoulddownload
+from communicator import peeraction, onlinepeers
+import onionrcrypto, onionrstorage
+from onionrblocks import onionrblacklist, storagecounter
+def download_blocks_from_communicator(comm_inst):
+    '''Use Onionr communicator instance to download blocks in the communicator's queue'''
+    assert isinstance(comm_inst, communicator.OnionrCommunicatorDaemon)
+    blacklist = onionrblacklist.OnionrBlackList()
+    storage_counter = storagecounter.StorageCounter()
+    LOG_SKIP_COUNT = 50 # for how many iterations we skip logging the counter
+    count: int = 0
+    metadata_validation_result: bool = False
+    # Iterate the block queue in the communicator
+    for blockHash in list(comm_inst.blockQueue):
+        count += 1
+        if len(comm_inst.onlinePeers) == 0:
+            break
+        triedQueuePeers = [] # List of peers we've tried for a block
+        try:
+            blockPeers = list(comm_inst.blockQueue[blockHash])
+        except KeyError:
+            blockPeers = []
+        removeFromQueue = True
+
+        if not shoulddownload.should_download(comm_inst, blockHash):
+            continue
+
+        if comm_inst.shutdown or not comm_inst.isOnline or storage_counter.is_full():
+            # Exit loop if shutting down or offline, or disk allocation reached
+            break
+        # Do not download blocks being downloaded
+        if blockHash in comm_inst.currentDownloading:
+            #logger.debug('Already downloading block %s...' % blockHash)
+            continue
+
+        comm_inst.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
+        if len(blockPeers) == 0:
+            peerUsed = onlinepeers.pick_online_peer(comm_inst)
+        else:
+            blockPeers = onionrcrypto.cryptoutils.random_shuffle(blockPeers)
+            peerUsed = blockPeers.pop(0)
+
+        if not comm_inst.shutdown and peerUsed.strip() != '':
+            logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
+        content = peeraction.peer_action(comm_inst, peerUsed, 'getdata/' + blockHash, max_resp_size=3000000) # block content from random peer (includes metadata)
+
+        if content != False and len(content) > 0:
+            try:
+                content = content.encode()
+            except AttributeError:
+                pass
+
+            realHash = onionrcrypto.hashers.sha3_hash(content)
+            try:
+                realHash = realHash.decode() # bytes on some versions for some reason
+            except AttributeError:
+                pass
+            if realHash == blockHash:
+                #content = content.decode() # decode here because sha3Hash needs bytes above
+                metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
+                metadata = metas[0]
+                try:
+                    metadata_validation_result = validatemetadata.validate_metadata(metadata, metas[2])
+                except onionrexceptions.DataExists:
+                    metadata_validation_result = False
+                if metadata_validation_result: # check if metadata is valid, and verify nonce
+                    if onionrcrypto.cryptoutils.verify_POW(content): # check if POW is enough/correct
+                        logger.info('Attempting to save block %s...' % blockHash[:12])
+                        try:
+                            onionrstorage.set_data(content)
+                        except onionrexceptions.DataExists:
+                            logger.warn('Data is already set for %s ' % (blockHash,))
+                        except onionrexceptions.DiskAllocationReached:
+                            logger.error('Reached disk allocation allowance, cannot save block %s.' % (blockHash,))
+                            removeFromQueue = False
+                        else:
+                            blockmetadb.add_to_block_DB(blockHash, dataSaved=True) # add block to meta db
+                            blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
+                    else:
+                        logger.warn('POW failed for block %s.' % (blockHash,))
+                else:
+                    if blacklist.inBlacklist(realHash):
+                        logger.warn('Block %s is blacklisted.' % (realHash,))
+                    else:
+                        logger.warn('Metadata for block %s is invalid.' % (blockHash,))
+                        blacklist.addToDB(blockHash)
+            else:
+                # if block didn't meet expected hash
+                tempHash = onionrcrypto.hashers.sha3_hash(content) # lazy hack, TODO use var
+                try:
+                    tempHash = tempHash.decode()
+                except AttributeError:
+                    pass
+                # Punish peer for sharing invalid block (not always malicious, but is bad regardless)
+                onionrpeers.PeerProfiles(peerUsed).addScore(-50)
+                if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253':
+                    # Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error.
+                    logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
+                else:
+                    removeFromQueue = False # Don't remove from queue if 404
+            if removeFromQueue:
+                try:
+                    del comm_inst.blockQueue[blockHash] # remove from block queue both if success or false
+                    if count == LOG_SKIP_COUNT:
+                        logger.info('%s blocks remaining in queue' % [len(comm_inst.blockQueue)], terminal=True)
+                        count = 0
+                except KeyError:
+                    pass
+        comm_inst.currentDownloading.remove(blockHash)
+    comm_inst.decrementThreadCount('getBlocks')

src/communicatorutils/downloadblocks/shoulddownload.py

@@ -0,0 +1,37 @@
+'''
+    Onionr - Private P2P Communication
+
+    Check if a block should be downloaded (if we already have it or its blacklisted or not)
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from coredb import blockmetadb
+from onionrblocks import onionrblacklist
+
+def should_download(comm_inst, block_hash):
+    blacklist = onionrblacklist.OnionrBlackList()
+    ret_data = True
+    if block_hash in blockmetadb.get_block_list(): # Dont download block we have
+        ret_data = False
+    else:
+        if blacklist.inBlacklist(block_hash): # Dont download blacklisted block
+            ret_data = False
+    if ret_data is False:
+        # Remove block from communicator queue if it shouldnt be downloaded
+        try:
+            del comm_inst.blockQueue[block_hash]
+        except KeyError:
+            pass
+    return ret_data

src/communicatorutils/housekeeping.py

@@ -0,0 +1,75 @@
+'''
+    Onionr - Private P2P Communication
+
+    Cleanup old Onionr blocks and forward secrecy keys using the communicator. Ran from a timer usually
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3
+import logger
+from onionrusers import onionrusers
+from onionrutils import epoch
+from coredb import blockmetadb, dbfiles
+import onionrstorage
+from onionrstorage import removeblock
+from onionrblocks import onionrblacklist
+
+def __remove_from_upload(comm_inst, block_hash: str):
+    try:
+        comm_inst.blocksToUpload.remove(block_hash)
+    except ValueError:
+        pass
+
+def clean_old_blocks(comm_inst):
+    '''Delete old blocks if our disk allocation is full/near full, and also expired blocks'''
+    blacklist = onionrblacklist.OnionrBlackList()
+    # Delete expired blocks
+    for bHash in blockmetadb.expiredblocks.get_expired_blocks():
+        blacklist.addToDB(bHash)
+        removeblock.remove_block(bHash)
+        onionrstorage.deleteBlock(bHash)
+        __remove_from_upload(comm_inst, bHash)
+        logger.info('Deleted block: %s' % (bHash,))
+
+    while comm_inst.storage_counter.is_full():
+        oldest = blockmetadb.get_block_list()[0]
+        blacklist.addToDB(oldest)
+        removeblock.remove_block(oldest)
+        onionrstorage.deleteBlock(oldest)
+        __remove_from_upload.remove(comm_inst, oldest)
+        logger.info('Deleted block: %s' % (oldest,))
+
+    comm_inst.decrementThreadCount('clean_old_blocks')
+
+def clean_keys(comm_inst):
+    '''Delete expired forward secrecy keys'''
+    conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
+    c = conn.cursor()
+    time = epoch.get_epoch()
+    deleteKeys = []
+
+    for entry in c.execute("SELECT * FROM forwardKeys WHERE expire <= ?", (time,)):
+        logger.debug('Forward key: %s' % entry[1])
+        deleteKeys.append(entry[1])
+
+    for key in deleteKeys:
+        logger.debug('Deleting forward key %s' % key)
+        c.execute("DELETE from forwardKeys where forwardKey = ?", (key,))
+    conn.commit()
+    conn.close()
+
+    onionrusers.deleteExpiredKeys()
+
+    comm_inst.decrementThreadCount('clean_keys')

src/communicatorutils/lookupadders.py

@@ -0,0 +1,55 @@
+'''
+    Onionr - Private P2P Communication
+
+    Lookup new peer transport addresses using the communicator
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import logger
+from onionrutils import stringvalidators
+from communicator import peeraction, onlinepeers
+from utils import gettransports
+def lookup_new_peer_transports_with_communicator(comm_inst):
+    logger.info('Looking up new addresses...')
+    tryAmount = 1
+    newPeers = []
+    transports = gettransports.get()
+
+    for i in range(tryAmount):
+        # Download new peer address list from random online peers
+        if len(newPeers) > 10000:
+            # Don't get new peers if we have too many queued up
+            break
+        peer = onlinepeers.pick_online_peer(comm_inst)
+        newAdders = peeraction.peer_action(comm_inst, peer, action='pex')
+        try:
+            newPeers = newAdders.split(',')
+        except AttributeError:
+            pass
+    else:
+        # Validate new peers are good format and not already in queue
+        invalid = []
+        for x in newPeers:
+            x = x.strip()
+            if not stringvalidators.validate_transport(x) or x in comm_inst.newPeers or x in transports:
+                # avoid adding if its our address
+                invalid.append(x)
+        for x in invalid:
+            try:
+                newPeers.remove(x)
+            except ValueError:
+                pass
+        comm_inst.newPeers.extend(newPeers)
+    comm_inst.decrementThreadCount('lookup_new_peer_transports_with_communicator')

src/communicatorutils/lookupblocks.py

@@ -0,0 +1,94 @@
+'''
+    Onionr - Private P2P Communication
+
+    Lookup new blocks with the communicator using a random connected peer
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import logger, onionrproofs
+from onionrutils import stringvalidators, epoch
+from communicator import peeraction, onlinepeers
+from coredb import blockmetadb
+from utils import reconstructhash
+from onionrblocks import onionrblacklist
+blacklist = onionrblacklist.OnionrBlackList()
+def lookup_blocks_from_communicator(comm_inst):
+    logger.info('Looking up new blocks')
+    tryAmount = 2
+    newBlocks = ''
+    existingBlocks = blockmetadb.get_block_list() # List of existing saved blocks
+    triedPeers = [] # list of peers we've tried this time around
+    maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
+    lastLookupTime = 0 # Last time we looked up a particular peer's list
+    new_block_count = 0
+    for i in range(tryAmount):
+        listLookupCommand = 'getblocklist' # This is defined here to reset it each time
+        if len(comm_inst.blockQueue) >= maxBacklog:
+            break
+        if not comm_inst.isOnline:
+            break
+        # check if disk allocation is used
+        if comm_inst.storage_counter.is_full():
+            logger.debug('Not looking up new blocks due to maximum amount of allowed disk space used')
+            break
+        peer = onlinepeers.pick_online_peer(comm_inst) # select random online peer
+        # if we've already tried all the online peers this time around, stop
+        if peer in triedPeers:
+            if len(comm_inst.onlinePeers) == len(triedPeers):
+                break
+            else:
+                continue
+        triedPeers.append(peer)
+
+        # Get the last time we looked up a peer's stamp to only fetch blocks since then.
+        # Saved in memory only for privacy reasons
+        try:
+            lastLookupTime = comm_inst.dbTimestamps[peer]
+        except KeyError:
+            lastLookupTime = 0
+        else:
+            listLookupCommand += '?date=%s' % (lastLookupTime,)
+        try:
+            newBlocks = peeraction.peer_action(comm_inst, peer, listLookupCommand) # get list of new block hashes
+        except Exception as error:
+            logger.warn('Could not get new blocks from %s.' % peer, error = error)
+            newBlocks = False
+        else:
+            comm_inst.dbTimestamps[peer] = epoch.get_rounded_epoch(roundS=60)
+        if newBlocks != False:
+            # if request was a success
+            for i in newBlocks.split('\n'):
+                if stringvalidators.validate_hash(i):
+                    i = reconstructhash.reconstruct_hash(i)
+                    # if newline seperated string is valid hash
+                    if not i in existingBlocks:
+                        # if block does not exist on disk and is not already in block queue
+                        if i not in comm_inst.blockQueue:
+                            if onionrproofs.hashMeetsDifficulty(i) and not blacklist.inBlacklist(i):
+                                if len(comm_inst.blockQueue) <= 1000000:
+                                    comm_inst.blockQueue[i] = [peer] # add blocks to download queue
+                                    new_block_count += 1
+                        else:
+                            if peer not in comm_inst.blockQueue[i]:
+                                if len(comm_inst.blockQueue[i]) < 10:
+                                    comm_inst.blockQueue[i].append(peer)
+    if new_block_count > 0:
+        block_string = ""
+        if new_block_count > 1:
+            block_string = "s"
+        logger.info('Discovered %s new block%s' % (new_block_count, block_string), terminal=True)
+        comm_inst.download_blocks_timer.count = int(comm_inst.download_blocks_timer.frequency * 0.99)
+    comm_inst.decrementThreadCount('lookup_blocks_from_communicator')
+    return

src/communicatorutils/netcheck.py

@@ -0,0 +1,43 @@
+'''
+    Onionr - Private P2P Communication
+
+    Determine if our node is able to use Tor based on the status of a communicator instance
+    and the result of pinging onion http servers
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import logger
+from utils import netutils
+from onionrutils import localcommand, epoch
+from . import restarttor
+def net_check(comm_inst):
+    '''Check if we are connected to the internet or not when we can't connect to any peers'''
+    rec = False # for detecting if we have received incoming connections recently
+    if len(comm_inst.onlinePeers) == 0:
+        try:
+            if (epoch.get_epoch() - int(localcommand.local_command('/lastconnect'))) <= 60:
+                comm_inst.isOnline = True
+                rec = True
+        except ValueError:
+            pass
+        if not rec and not netutils.checkNetwork(torPort=comm_inst.proxyPort):
+            if not comm_inst.shutdown:
+                logger.warn('Network check failed, are you connected to the Internet, and is Tor working?', terminal=True)
+                restarttor.restart(comm_inst)
+                comm_inst.offlinePeers = []
+            comm_inst.isOnline = False
+        else:
+            comm_inst.isOnline = True
+    comm_inst.decrementThreadCount('net_check')

src/communicatorutils/onionrcommunicatortimers.py

@@ -0,0 +1,79 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file contains timer control for the communicator
+'''
+from __future__ import annotations  # thank you python, very cool
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import uuid
+import threading
+
+import onionrexceptions, logger
+
+from typing import TYPE_CHECKING
+from typing import Callable, NewType, Iterable
+if TYPE_CHECKING:
+    from communicator import OnionrCommunicatorDaemon
+
+CallFreqSeconds = NewType('CallFreqSeconds', int)
+
+class OnionrCommunicatorTimers:
+    def __init__(self, daemon_inst: OnionrCommunicatorDaemon, 
+                timer_function: Callable, frequency: CallFreqSeconds, 
+                make_thread:bool=True, thread_amount:int=1, max_threads:int=5, 
+                requires_peer:bool=False, my_args:Iterable=[]):
+        self.timer_function = timer_function
+        self.frequency = frequency
+        self.thread_amount = thread_amount
+        self.make_thread = make_thread
+        self.requires_peer = requires_peer
+        self.daemon_inst = daemon_inst
+        self.max_threads = max_threads
+        self.args = my_args
+
+        self.daemon_inst.timers.append(self)
+        self.count = 0
+
+    def processTimer(self):
+
+        # mark how many instances of a thread we have (decremented at thread end)
+        try:
+            self.daemon_inst.threadCounts[self.timer_function.__name__]
+        except KeyError:
+            self.daemon_inst.threadCounts[self.timer_function.__name__] = 0
+
+        # execute thread if it is time, and we are not missing *required* online peer
+        if self.count == self.frequency and not self.daemon_inst.shutdown:
+            try:
+                if self.requires_peer and len(self.daemon_inst.onlinePeers) == 0:
+                    raise onionrexceptions.OnlinePeerNeeded
+            except onionrexceptions.OnlinePeerNeeded:
+                return
+            else:
+                if self.make_thread:
+                    for i in range(self.thread_amount):
+                        if self.daemon_inst.threadCounts[self.timer_function.__name__] >= self.max_threads:
+                            logger.debug('%s is currently using the maximum number of threads, not starting another.' % self.timer_function.__name__)
+                        else:
+                            self.daemon_inst.threadCounts[self.timer_function.__name__] += 1
+                            newThread = threading.Thread(target=self.timer_function, args=self.args, daemon=True, 
+                                                        name=self.timer_function.__name__ + ' - ' + str(uuid.uuid4()))
+                            newThread.start()
+                else:
+                    self.timer_function()
+            self.count = -1 # negative 1 because its incremented at bottom
+        self.count += 1

src/communicatorutils/proxypicker.py

@@ -0,0 +1,26 @@
+'''
+    Onionr - Private P2P Communication
+
+    Just picks a proxy to use based on a peer's address
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+def pick_proxy(peer_address):
+    if peer_address.endswith('.onion'):
+        return 'tor'
+    elif peer_address.endswith('.i2p'):
+        return 'i2p'
+    raise ValueError(f"Peer address was not string ending with acceptable value: {peer_address}")

src/communicatorutils/restarttor.py

@@ -0,0 +1,5 @@
+import netcontroller
+def restart(comm_inst):
+    net = comm_inst.shared_state.get(netcontroller.NetController)
+    net.killTor()
+    net.startTor()

src/communicatorutils/servicecreator.py

@@ -0,0 +1,45 @@
+'''
+    Onionr - Private P2P Communication
+
+    Creates an onionr direct connection service by scanning all connection blocks
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import communicator
+from onionrblocks import onionrblockapi
+import logger
+from onionrutils import stringvalidators, bytesconverter
+from coredb import blockmetadb
+from onionrservices import server_exists
+def service_creator(daemon):
+    assert isinstance(daemon, communicator.OnionrCommunicatorDaemon)
+    
+    # Find socket connection blocks
+    # TODO cache blocks and only look at recently received ones
+    con_blocks = blockmetadb.get_blocks_by_type('con')
+    for b in con_blocks:
+        if not b in daemon.active_services:
+            bl = onionrblockapi.Block(b, decrypt=True)
+            bs = bytesconverter.bytes_to_str(bl.bcontent) + '.onion'
+            if server_exists(bl.signer):
+                continue
+            if stringvalidators.validate_pub_key(bl.signer) and stringvalidators.validate_transport(bs):
+                signer = bytesconverter.bytes_to_str(bl.signer)
+                daemon.active_services.append(b)
+                daemon.active_services.append(signer)
+                if not daemon.services.create_server(signer, bs, daemon):
+                    daemon.active_services.remove(b)
+                    daemon.active_services.remove(signer)
+    daemon.decrementThreadCount('service_creator')

src/communicatorutils/uploadblocks/__init__.py

@@ -0,0 +1,90 @@
+'''
+    Onionr - Private P2P Communication
+
+    Upload blocks in the upload queue to peers from the communicator
+'''
+from __future__ import annotations
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from typing import Union, TYPE_CHECKING
+import logger
+from communicatorutils import proxypicker
+import onionrexceptions
+from onionrblocks import onionrblockapi as block
+from onionrutils import localcommand, stringvalidators, basicrequests
+from communicator import onlinepeers
+import onionrcrypto
+from . import sessionmanager
+
+def upload_blocks_from_communicator(comm_inst: OnionrCommunicatorDaemon):
+    """Accepts a communicator instance and uploads blocks from its upload queue"""
+    """when inserting a block, we try to upload
+     it to a few peers to add some deniability & increase functionality"""
+    TIMER_NAME = "upload_blocks_from_communicator"
+
+    session_manager: sessionmanager.BlockUploadSessionManager = comm_inst.shared_state.get(sessionmanager.BlockUploadSessionManager)
+    triedPeers = []
+    finishedUploads = []
+    comm_inst.blocksToUpload = onionrcrypto.cryptoutils.random_shuffle(comm_inst.blocksToUpload)
+    if len(comm_inst.blocksToUpload) != 0:
+        for bl in comm_inst.blocksToUpload:
+            if not stringvalidators.validate_hash(bl):
+                logger.warn('Requested to upload invalid block', terminal=True)
+                comm_inst.decrementThreadCount(TIMER_NAME)
+                return
+            session = session_manager.add_session(bl)
+            for i in range(min(len(comm_inst.onlinePeers), 6)):
+                peer = onlinepeers.pick_online_peer(comm_inst)
+                try:
+                    session.peer_exists[peer]
+                    continue
+                except KeyError:
+                    pass
+                try:
+                    if session.peer_fails[peer] > 3: continue
+                except KeyError:
+                    pass
+                if peer in triedPeers: continue
+                triedPeers.append(peer)
+                url = f'http://{peer}/upload'
+                try:
+                    data = block.Block(bl).getRaw()
+                except onionrexceptions.NoDataAvailable:
+                    finishedUploads.append(bl)
+                    break
+                proxyType = proxypicker.pick_proxy(peer)
+                logger.info(f"Uploading block {bl[:8]} to {peer}", terminal=True)
+                resp = basicrequests.do_post_request(url, data=data, proxyType=proxyType, content_type='application/octet-stream')
+                if not resp == False:
+                    if resp == 'success':
+                        session.success()
+                        session.peer_exists[peer] = True
+                    elif resp == 'exists':
+                        session.success()
+                        session.peer_exists[peer] = True
+                    else:
+                        session.fail()
+                        session.fail_peer(peer)
+                        comm_inst.getPeerProfileInstance(peer).addScore(-5)
+                        logger.warn(f'Failed to upload {bl[:8]}, reason: {resp[:15]}', terminal=True)
+                else:
+                    session.fail()
+        session_manager.clean_session()
+    for x in finishedUploads:
+        try:
+            comm_inst.blocksToUpload.remove(x)
+        except ValueError:
+            pass
+    comm_inst.decrementThreadCount(TIMER_NAME)

src/communicatorutils/uploadblocks/session.py

@@ -0,0 +1,54 @@
+"""
+    Onionr - Private P2P Communication
+
+    Virtual upload "sessions" for blocks
+"""
+from __future__ import annotations
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+from typing import Union
+
+from onionrutils import stringvalidators
+from onionrutils import bytesconverter
+from onionrutils import epoch
+from utils import reconstructhash
+
+class UploadSession:
+    """Manages statistics for an Onionr block upload session
+    
+    accepting a block hash (incl. unpadded) as an argument""" 
+    def __init__(self, block_hash: Union[str, bytes]):
+        block_hash = bytesconverter.bytes_to_str(block_hash)
+        block_hash = reconstructhash.reconstruct_hash(block_hash)
+        if not stringvalidators.validate_hash(block_hash): raise ValueError
+
+        self.start_time = epoch.get_epoch()
+        self.block_hash = reconstructhash.deconstruct_hash(block_hash)
+        self.total_fail_count: int = 0
+        self.total_success_count: int = 0
+        self.peer_fails = {}
+        self.peer_exists = {}
+    
+    def fail_peer(self, peer):
+        try:
+            self.peer_fails[peer] += 1
+        except KeyError:
+            self.peer_fails[peer] = 0
+    
+    def fail(self):
+        self.total_fail_count += 1
+    
+    def success(self):
+        self.total_success_count += 1

src/communicatorutils/uploadblocks/sessionmanager.py

@@ -0,0 +1,95 @@
+"""
+    Onionr - Private P2P Communication
+
+    Manager for upload 'sessions'
+"""
+from __future__ import annotations
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+from typing import Iterable, Union
+
+from onionrutils import bytesconverter
+from onionrutils import localcommand
+from etc import onionrvalues
+from etc import waitforsetvar
+from utils import reconstructhash
+
+from . import session
+
+class BlockUploadSessionManager:
+    """Holds block UploadSession instances. Optionally accepts iterable of sessions to added on init
+
+    Arguments: old_session: iterable of old UploadSession objects"""
+    def __init__(self, old_sessions:Iterable=None):
+        #self._too_many: TooMany = None
+        if old_sessions is None:
+            self.sessions = []
+        else:
+            self.sessions = old_session
+    
+    def add_session(self, session_or_block: Union(str, bytes, session.UploadSession))->session.UploadSession:
+        """Create (or add existing) block upload session from a str/bytes block hex hash, existing UploadSession"""
+        if isinstance(session_or_block, session.UploadSession): 
+            if not session_or_block in self.sessions:
+                self.sessions.append(session_or_block)
+            return session_or_block
+        try:
+            return self.get_session(session_or_block)
+        except KeyError:
+            pass
+        # convert bytes hash to str
+        if isinstance(session_or_block, bytes): session_or_block = bytesconverter.bytes_to_str(session_or_block)
+        # intentionally not elif
+        if isinstance(session_or_block, str):
+            new_session = session.UploadSession(session_or_block)
+            self.sessions.append(new_session)
+            return new_session
+
+    def get_session(self, block_hash: Union(str, bytes))->session.UploadSession:
+        block_hash = reconstructhash.deconstruct_hash(bytesconverter.bytes_to_str(block_hash))
+        for session in self.sessions:
+            if session.block_hash == block_hash: return session
+        raise KeyError
+
+    def clean_session(self, specific_session: Union[str, UploadSession]=None):
+        comm_inst: OnionrCommunicatorDaemon = self._too_many.get_by_string("OnionrCommunicatorDaemon")
+        sessions_to_delete = []
+        if comm_inst.getUptime() < 120: return
+        onlinePeerCount = len(comm_inst.onlinePeers)
+        
+        # If we have no online peers right now,
+        if onlinePeerCount == 0: return
+
+        for session in self.sessions:
+            # if over 50% of peers that were online for a session have become unavailable, don't kill sessions
+            if session.total_success_count > onlinePeerCount:
+                if onlinePeerCount / session.total_success_count >= 0.5: return
+            # Clean sessions if they have uploaded to enough online peers
+            if session.total_success_count <= 0: continue
+            if (session.total_success_count / onlinePeerCount) >= onionrvalues.MIN_BLOCK_UPLOAD_PEER_PERCENT:
+                sessions_to_delete.append(session)
+        for session in sessions_to_delete:
+            self.sessions.remove(session)
+            # TODO cleanup to one round of search
+            # Remove the blocks from the sessions, upload list, and waitforshare list
+            try:
+                comm_inst.blocksToUpload.remove(reconstructhash.reconstruct_hash(session.block_hash))
+            except ValueError:
+                pass
+            try:
+                comm_inst.blocksToUpload.remove(session.block_hash)
+            except ValueError:
+                pass
+            localcommand.local_command('waitforshare/{session.block_hash}')

src/config.py

@@ -0,0 +1,146 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file deals with configuration management.
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import os, json, logger
+import filepaths
+
+_configfile = filepaths.config_file
+_config = {}
+
+def get(key, default = None, save = False):
+    '''
+        Gets the key from configuration, or returns `default`
+    '''
+
+    key = str(key).split('.')
+    data = _config
+
+    last = key.pop()
+
+    for item in key:
+        if (not item in data) or (not type(data[item]) == dict):
+            return default
+        data = data[item]
+
+    if not last in data:
+        if save:
+            set(key, default, savefile = True)
+        return default
+
+    return data[last]
+
+def set(key, value = None, savefile = False):
+    '''
+        Sets the key in configuration to `value`
+    '''
+
+    global _config
+
+    key = str(key).split('.')
+    data = _config
+
+    last = key.pop()
+
+    for item in key:
+        if (not item in data) or (not type(data[item]) == dict):
+            data[item] = dict()
+        data = data[item]
+
+    if value is None:
+        del data[last]
+    else:
+        data[last] = value
+
+    if savefile:
+        save()
+
+def is_set(key):
+    key = str(key).split('.')
+    data = _config
+
+    last = key.pop()
+
+    for item in key:
+        if (not item in data) or (not type(data[item]) == dict):
+            return False
+        data = data[item]
+
+    if not last in data:
+        return False
+
+    return True
+
+def check():
+    '''
+        Checks if the configuration file exists, creates it if not
+    '''
+
+    if not os.path.exists(os.path.dirname(get_config_file())):
+        os.makedirs(os.path.dirname(get_config_file()))
+
+def save():
+    '''
+        Saves the configuration data to the configuration file
+    '''
+
+    check()
+    try:
+        with open(get_config_file(), 'w', encoding="utf8") as configfile:
+            json.dump(get_config(), configfile, indent=2)
+    except json.JSONDecodeError:
+        logger.warn('Failed to write to configuration file.')
+
+def reload():
+    '''
+        Reloads the configuration data in memory from the file
+    '''
+    check()
+    try:
+        with open(get_config_file(), 'r', encoding="utf8") as configfile:
+            set_config(json.loads(configfile.read()))
+    except (FileNotFoundError, json.JSONDecodeError) as e:
+        pass
+        #logger.debug('Failed to parse configuration file.')
+
+def get_config():
+    '''
+        Gets the entire configuration as an array
+    '''
+    return _config
+
+def set_config(config):
+    '''
+        Sets the configuration to the array in arguments
+    '''
+    global _config
+    _config = config
+
+def get_config_file():
+    '''
+        Returns the absolute path to the configuration file
+    '''
+    return _configfile
+
+def set_config_file(configfile):
+    '''
+        Sets the path to the configuration file
+    '''
+    global _configfile
+    _configfile = os.abs.abspath(configfile)

src/coredb/__init__.py

@@ -0,0 +1 @@
+from . import keydb, blockmetadb, daemonqueue

src/coredb/blockmetadb/__init__.py

@@ -0,0 +1,83 @@
+'''
+    Onionr - Private P2P Communication
+
+    This module works with information relating to blocks stored on the node
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3
+
+from etc import onionrvalues
+from . import expiredblocks, updateblockinfo, add
+from .. import dbfiles
+
+update_block_info = updateblockinfo.update_block_info
+add_to_block_DB = add.add_to_block_DB
+def get_block_list(dateRec = None, unsaved = False):
+    '''
+        Get list of our blocks
+    '''
+    if dateRec == None:
+        dateRec = 0
+
+    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    c = conn.cursor()
+
+    execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
+    args = (dateRec,)
+    rows = list()
+    for row in c.execute(execute, args):
+        for i in row:
+            rows.append(i)
+    conn.close()
+    return rows
+
+def get_block_date(blockHash):
+    '''
+        Returns the date a block was received
+    '''
+
+    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    c = conn.cursor()
+
+    execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
+    args = (blockHash,)
+    for row in c.execute(execute, args):
+        for i in row:
+            return int(i)
+    conn.close()
+    return None
+
+def get_blocks_by_type(blockType, orderDate=True):
+    '''
+        Returns a list of blocks by the type
+    '''
+
+    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    c = conn.cursor()
+
+    if orderDate:
+        execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;'
+    else:
+        execute = 'SELECT hash FROM hashes WHERE dataType=?;'
+
+    args = (blockType,)
+    rows = list()
+
+    for row in c.execute(execute, args):
+        for i in row:
+            rows.append(i)
+    conn.close()
+    return rows

src/coredb/blockmetadb/add.py

@@ -0,0 +1,43 @@
+'''
+    Onionr - Private P2P Communication
+
+    Add an entry to the block metadata database
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import os, sqlite3, secrets
+from onionrutils import epoch, blockmetadata
+from etc import onionrvalues
+from .. import dbfiles
+def add_to_block_DB(newHash, selfInsert=False, dataSaved=False):
+    '''
+        Add a hash value to the block db
+
+        Should be in hex format!
+    '''
+
+    if blockmetadata.has_block(newHash):
+        return
+    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    c = conn.cursor()
+    currentTime = epoch.get_epoch() + secrets.randbelow(301)
+    if selfInsert or dataSaved:
+        selfInsert = 1
+    else:
+        selfInsert = 0
+    data = (newHash, currentTime, '', selfInsert)
+    c.execute('INSERT INTO hashes (hash, dateReceived, dataType, dataSaved) VALUES(?, ?, ?, ?);', data)
+    conn.commit()
+    conn.close()

src/coredb/blockmetadb/expiredblocks.py

@@ -0,0 +1,39 @@
+'''
+    Onionr - Private P2P Communication
+
+    Get a list of expired blocks still stored
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3
+from onionrutils import epoch
+from .. import dbfiles
+from etc import onionrvalues
+
+def get_expired_blocks():
+    '''Returns a list of expired blocks'''
+    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    c = conn.cursor()
+    date = int(epoch.get_epoch())
+
+    compiled = (date,)
+    execute = 'SELECT hash FROM hashes WHERE expire <= ? ORDER BY dateReceived;'
+
+    rows = list()
+    for row in c.execute(execute, compiled):
+        for i in row:
+            rows.append(i)
+    conn.close()
+    return rows

src/coredb/blockmetadb/updateblockinfo.py

@@ -0,0 +1,50 @@
+'''
+    Onionr - Private P2P Communication
+
+    Update block information in the metadata database by a field name
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3
+from .. import dbfiles
+from etc import onionrvalues
+def update_block_info(hash, key, data):
+    '''
+        sets info associated with a block
+
+        hash         - the hash of a block
+        dateReceived - the date the block was recieved, not necessarily when it was created
+        decrypted    - if we can successfully decrypt the block (does not describe its current state)
+        dataType     - data type of the block
+        dataFound    - if the data has been found for the block
+        dataSaved    - if the data has been saved for the block
+        sig    - optional signature by the author (not optional if author is specified)
+        author       - multi-round partial sha3-256 hash of authors public key
+        dateClaimed  - timestamp claimed inside the block, only as trustworthy as the block author is
+        expire       - expire date for a block
+    '''
+    if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 
+                  'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'):
+        raise ValueError('Key must be in the allowed list')
+
+    conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    c = conn.cursor()
+    args = (data, hash)
+    # Unfortunately, not really possible
+    c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args)
+    conn.commit()
+    conn.close()
+
+    return True

src/coredb/daemonqueue/__init__.py

@@ -0,0 +1,93 @@
+'''
+    Onionr - Private P2P Communication
+
+    Write and read the daemon queue, which is how messages are passed into the onionr daemon in a more
+    direct way than the http api
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3, os
+from onionrplugins import onionrevents as events
+from onionrutils import localcommand, epoch
+from .. import dbfiles
+from onionrsetup import dbcreator
+from etc import onionrvalues
+
+def daemon_queue()->str:
+    '''
+        Gives commands to the communication proccess/daemon by reading an sqlite3 database
+
+        This function intended to be used by the client. Queue to exchange data between "client" and server.
+    '''
+
+    retData = False
+    if not os.path.exists(dbfiles.daemon_queue_db):
+        dbcreator.createDaemonDB()
+    else:
+        conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+        c = conn.cursor()
+        try:
+            for row in c.execute('SELECT command, data, date, min(ID), responseID FROM commands group by id'):
+                retData = row
+                break
+        except sqlite3.OperationalError:
+            dbcreator.createDaemonDB()
+        else:
+            if retData != False:
+                c.execute('DELETE FROM commands WHERE id=?;', (retData[3],))
+        conn.commit()
+        conn.close()
+
+    return retData
+
+def daemon_queue_add(command: str, data='', responseID: str =''):
+    '''
+        Add a command to the daemon queue, used by the communication daemon (communicator.py)
+    '''
+
+    retData = True
+
+    date = epoch.get_epoch()
+    conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    c = conn.cursor()
+    t = (command, data, date, responseID)
+    try:
+        c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t)
+        conn.commit()
+    except sqlite3.OperationalError:
+        retData = False
+        daemon_queue()
+    conn.close()
+    return retData
+
+def daemon_queue_get_response(responseID=''):
+    '''
+        Get a response sent by communicator to the API, by requesting to the API
+    '''
+    if len(responseID) == 0: raise ValueError('ResponseID should not be empty')
+    resp = localcommand.local_command(dbfiles.daemon_queue_db, 'queueResponse/' + responseID)
+    return resp
+
+def clear_daemon_queue():
+    '''
+        Clear the daemon queue (somewhat dangerous)
+    '''
+    conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    c = conn.cursor()
+
+    c.execute('DELETE FROM commands;')
+    conn.commit()
+
+    conn.close()

src/coredb/dbfiles.py

@@ -0,0 +1,12 @@
+from utils import identifyhome
+import filepaths
+home = identifyhome.identify_home()
+if not home.endswith('/'): home += '/'
+
+block_meta_db = '%sblock-metadata.db' % (home)
+block_data_db = '%s/block-data.db' % (filepaths.block_data_location,)
+daemon_queue_db = '%sdaemon-queue.db' % (home,)
+address_info_db = '%saddress.db' % (home,)
+user_id_info_db = '%susers.db' % (home,)
+forward_keys_db = '%sforward-keys.db' % (home,)
+blacklist_db = '%sblacklist.db' % (home,)

src/coredb/keydb/__init__.py

@@ -0,0 +1 @@
+from . import addkeys, listkeys, removekeys, userinfo, transportinfo

src/coredb/keydb/addkeys.py

@@ -0,0 +1,96 @@
+'''
+    Onionr - Private P2P Communication
+
+    add user keys or transport addresses
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3
+from onionrplugins import onionrevents as events
+from onionrutils import stringvalidators
+from . import listkeys
+from utils import gettransports
+from .. import dbfiles
+import onionrcrypto
+from etc import onionrvalues
+
+def add_peer(peerID, name=''):
+    '''
+        Adds a public key to the key database (misleading function name)
+    '''
+    if peerID in listkeys.list_peers() or peerID == onionrcrypto.pub_key:
+        raise ValueError("specified id is already known")
+
+    # This function simply adds a peer to the DB
+    if not stringvalidators.validate_pub_key(peerID):
+        return False
+
+    #events.event('pubkey_add', data = {'key': peerID}, onionr = core_inst.onionrInst)
+
+    conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    hashID = ""
+    c = conn.cursor()
+    t = (peerID, name, 'unknown', hashID, 0)
+
+    for i in c.execute("SELECT * FROM peers WHERE id = ?;", (peerID,)):
+        try:
+            if i[0] == peerID:
+                conn.close()
+                return False
+        except ValueError:
+            pass
+        except IndexError:
+            pass
+    c.execute('INSERT INTO peers (id, name, dateSeen, hashID, trust) VALUES(?, ?, ?, ?, ?);', t)
+    conn.commit()
+    conn.close()
+
+    return True
+
+def add_address(address):
+    '''
+        Add an address to the address database (only tor currently)
+    '''
+
+    if type(address) is None or len(address) == 0:
+        return False
+    if stringvalidators.validate_transport(address):
+        if address in gettransports.get():
+            return False
+        conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+        c = conn.cursor()
+        # check if address is in database
+        # this is safe to do because the address is validated above, but we strip some chars here too just in case
+        address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '')
+        for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)):
+            try:
+                if i[0] == address:
+                    conn.close()
+                    return False
+            except ValueError:
+                pass
+            except IndexError:
+                pass
+
+        t = (address, 1)
+        c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t)
+        conn.commit()
+        conn.close()
+
+        #events.event('address_add', data = {'address': address}, onionr = core_inst.onionrInst)
+
+        return True
+    else:
+        return False

src/coredb/keydb/listkeys.py

@@ -0,0 +1,86 @@
+'''
+    Onionr - Private P2P Communication
+
+    get lists for user keys or transport addresses
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3
+import logger
+from onionrutils import epoch
+from etc import onionrvalues
+from .. import dbfiles
+from . import userinfo, transportinfo
+def list_peers(randomOrder=True, getPow=False, trust=0):
+    '''
+        Return a list of public keys (misleading function name)
+
+        randomOrder determines if the list should be in a random order
+        trust sets the minimum trust to list
+    '''
+    conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    c = conn.cursor()
+
+    payload = ''
+
+    if trust not in (0, 1, 2):
+        logger.error('Tried to select invalid trust.')
+        return
+
+    if randomOrder:
+        payload = 'SELECT * FROM peers WHERE trust >= ? ORDER BY RANDOM();'
+    else:
+        payload = 'SELECT * FROM peers WHERE trust >= ?;'
+
+    peerList = []
+
+    for i in c.execute(payload, (trust,)):
+        try:
+            if len(i[0]) != 0:
+                if getPow:
+                    peerList.append(i[0] + '-' + i[1])
+                else:
+                    peerList.append(i[0])
+        except TypeError:
+            pass
+
+    conn.close()
+
+    return peerList
+
+def list_adders(randomOrder=True, i2p=True, recent=0):
+    '''
+        Return a list of transport addresses
+    '''
+    conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    c = conn.cursor()
+    if randomOrder:
+        addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();')
+    else:
+        addresses = c.execute('SELECT * FROM adders;')
+    addressList = []
+    for i in addresses:
+        if len(i[0].strip()) == 0:
+            continue
+        addressList.append(i[0])
+    conn.close()
+    testList = list(addressList) # create new list to iterate
+    for address in testList:
+        try:
+            if recent > 0 and (epoch.get_epoch() - transportinfo.get_address_info(address, 'lastConnect')) > recent:
+                raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0
+        except TypeError:
+            addressList.remove(address)
+    return addressList

src/coredb/keydb/removekeys.py

@@ -0,0 +1,60 @@
+'''
+    Onionr - Private P2P Communication
+
+    Remove a transport address but don't ban them
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3
+from onionrplugins import onionrevents as events
+from onionrutils import stringvalidators
+from onionrutils import mnemonickeys
+from .. import dbfiles
+from etc import onionrvalues
+
+def remove_address(address):
+    '''
+        Remove an address from the address database
+    '''
+
+    if stringvalidators.validate_transport(address):
+        conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+        c = conn.cursor()
+        t = (address,)
+        c.execute('Delete from adders where address=?;', t)
+        conn.commit()
+        conn.close()
+
+        #events.event('address_remove', data = {'address': address}, onionr = core_inst.onionrInst)
+        return True
+    else:
+        return False
+
+def remove_user(pubkey: str)->bool:
+    '''
+        Remove a user from the user database
+    '''
+    pubkey = mnemonickeys.get_base32(pubkey)
+    if stringvalidators.validate_pub_key(pubkey):
+        conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+        c = conn.cursor()
+        t = (pubkey,)
+        c.execute('Delete from peers where id=?;', t)
+        conn.commit()
+        conn.close()
+
+        return True
+    else:
+        return False

src/coredb/keydb/transportinfo.py

@@ -0,0 +1,75 @@
+'''
+    Onionr - Private P2P Communication
+
+    get or set transport address meta information
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3
+from .. import dbfiles
+from etc import onionrvalues
+
+def get_address_info(address, info):
+    '''
+        Get info about an address from its database entry
+
+        address text, 0
+        type int, 1
+        knownPeer text, 2
+        speed int, 3
+        success int, 4
+        powValue    5
+        failure int 6
+        lastConnect 7
+        trust       8
+        introduced  9
+    '''
+
+    conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    c = conn.cursor()
+
+    command = (address,)
+    infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'powValue': 5, 'failure': 6, 'lastConnect': 7, 'trust': 8, 'introduced': 9}
+    info = infoNumbers[info]
+    iterCount = 0
+    retVal = ''
+
+    for row in c.execute('SELECT * FROM adders WHERE address=?;', command):
+        for i in row:
+            if iterCount == info:
+                retVal = i
+                break
+            else:
+                iterCount += 1
+    conn.close()
+
+    return retVal
+
+def set_address_info(address, key, data):
+    '''
+        Update an address for a key
+    '''
+
+    conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    c = conn.cursor()
+
+    command = (data, address)
+
+    if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'failure', 'powValue', 'lastConnect', 'lastConnectAttempt', 'trust', 'introduced'):
+        raise ValueError("Got invalid database key when setting address info, must be in whitelist")
+    else:
+        c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command)
+        conn.commit()
+    conn.close()

src/coredb/keydb/userinfo.py

@@ -0,0 +1,73 @@
+'''
+    Onionr - Private P2P Communication
+
+    get or set information about a user id
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sqlite3
+from .. import dbfiles
+from etc import onionrvalues
+
+def get_user_info(peer, info):
+    '''
+        Get info about a peer from their database entry
+
+        id text             0
+        name text,          1
+        adders text,        2
+        dateSeen not null,  3
+        trust int           4
+        hashID text         5
+    '''
+    conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    c = conn.cursor()
+
+    command = (peer,)
+    infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'trust': 4, 'hashID': 5}
+    info = infoNumbers[info]
+    iterCount = 0
+    retVal = ''
+
+    for row in c.execute('SELECT * FROM peers WHERE id=?;', command):
+        for i in row:
+            if iterCount == info:
+                retVal = i
+                break
+            else:
+                iterCount += 1
+
+    conn.close()
+
+    return retVal
+
+def set_peer_info(peer, key, data):
+    '''
+        Update a peer for a key
+    '''
+
+    conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
+    c = conn.cursor()
+
+    command = (data, peer)
+
+    if key not in ('id', 'name', 'pubkey', 'forwardKey', 'dateSeen', 'trust'):
+        raise ValueError("Got invalid database key when setting peer info")
+
+    c.execute('UPDATE peers SET ' + key + ' = ? WHERE id=?', command)
+    conn.commit()
+    conn.close()
+
+set_user_info = set_peer_info

src/etc/README.md

@@ -0,0 +1,9 @@
+# etc
+
+Files that don't really fit anywhere else, but aren't used very frequently.
+
+## Files
+
+humanreadabletime.py: take integer seconds and return a human readable time string
+
+onionrvalues.py: spec values for onionr blocks and other things

src/etc/cleanup/__init__.py

@@ -0,0 +1,32 @@
+"""
+    Onionr - Private P2P Communication
+
+    cleanup files
+"""
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+import os, filepaths
+
+def _safe_remove(path):
+    try:
+        os.remove(path)
+    except FileNotFoundError:
+        pass
+
+def delete_run_files():
+     _safe_remove(filepaths.public_API_host_file)
+     _safe_remove(filepaths.private_API_host_file)
+     _safe_remove(filepaths.daemon_mark_file)
+     _safe_remove(filepaths.lock_file)

src/etc/dependencycheck.py

@@ -0,0 +1 @@
+from urllib3.contrib.socks import SOCKSProxyManager # noqa

src/etc/humanreadabletime.py

@@ -0,0 +1,38 @@
+'''
+    Onionr - Private P2P Communication
+
+    human_readable_time takes integer seconds and returns a human readable string
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+def human_readable_time(seconds):
+    build = ''
+
+    units = {
+        'year' : 31557600,
+        'month' : (31557600 / 12),
+        'day' : 86400,
+        'hour' : 3600,
+        'minute' : 60,
+        'second' : 1
+    }
+
+    for unit in units:
+        amnt_unit = int(seconds / units[unit])
+        if amnt_unit >= 1:
+            seconds -= amnt_unit * units[unit]
+            build += '%s %s' % (amnt_unit, unit) + ('s' if amnt_unit != 1 else '') + ' '
+
+    return build.strip()

src/etc/onionrvalues.py

@@ -0,0 +1,68 @@
+"""
+    Onionr - Private P2P Communication
+
+    This file defines values and requirements used by Onionr
+"""
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+import platform
+import os
+
+import filepaths
+
+DENIABLE_PEER_ADDRESS = "OVPCZLOXD6DC5JHX4EQ3PSOGAZ3T24F75HQLIUZSDSMYPEOXCPFA"
+PASSWORD_LENGTH = 25
+ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net'
+ONIONR_VERSION = '0.1.0' # for debugging and stuff
+ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
+API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
+MIN_PY_VERSION = 7 # min version of 7 so we can take advantage of non-cyclic type hints
+DEVELOPMENT_MODE = False
+"""limit type length for a block (soft enforced, ignored if invalid but block still stored)."""
+MAX_BLOCK_TYPE_LENGTH = 15
+"""limit clock timestamp for new blocks to be skewed in the future in seconds,
+2 minutes to allow plenty of time for slow block insertion and slight clock inaccuracies"""
+MAX_BLOCK_CLOCK_SKEW = 120
+"""Onionr user IDs are ed25519 keys, which are always 32 bytes in length"""
+MAIN_PUBLIC_KEY_SIZE = 32
+ORIG_RUN_DIR_ENV_VAR = 'ORIG_ONIONR_RUN_DIR'
+
+DATABASE_LOCK_TIMEOUT = 60
+
+# Block creation anonymization requirements
+MIN_BLOCK_UPLOAD_PEER_PERCENT = 0.1
+
+# Begin OnionrValues migrated values
+"""Make announce take a few seconds (on average) to compute to discourage excessive node announcements"""
+ANNOUNCE_POW = 6
+"""30 days is plenty of time for someone to decide to renew a block"""
+DEFAULT_EXPIRE = 2592000
+# Metadata header section length limits, in bytes
+BLOCK_METADATA_LENGTHS = {'meta': 1000, 'sig': 200, 'signer': 200, 'time': 10, 'pow': 1000, 'encryptType': 4, 'expire': 14}
+
+"""Public key that signs MOTD messages shown in the web UI"""
+MOTD_SIGN_KEY = "TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ"
+
+"""Public key that signs update notifications."""
+UPDATE_SIGN_KEY = "TRH763JURNY47QPBTTQ4LLPYCYQK6Q5YA33R6GANKZK5C5DKCIGQ"
+
+platform = platform.system()
+if platform == 'Windows':
+    SCRIPT_NAME = 'run-windows.bat'
+else:
+    if os.path.exists(filepaths.daemon_mark_file):
+        SCRIPT_NAME = 'start-daemon.sh'
+    else:
+        SCRIPT_NAME = 'onionr.sh'

src/etc/powchoice.py

@@ -0,0 +1,27 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file does determinations for what proof of work module should be used
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import platform
+def use_subprocess(config_inst):
+    use = True
+    if not config_inst.get('general.use_subprocess_pow_if_possible', True):
+        use = False
+    if 'Windows' == platform.system():
+        use = False
+    return use

src/etc/waitforsetvar.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+from typing import Union, Generic
+from gevent import sleep
+def wait_for_set_var(obj, attribute, sleep_seconds: Union[int, float]=0):
+    """Wait for an object to get an attribute with an optional sleep time"""
+    while not hasattr(obj, attribute):
+        if hasattr(obj, attribute): break
+        if sleep_seconds > 0: sleep(sleep_seconds)

src/filepaths/__init__.py

@@ -0,0 +1,31 @@
+from utils import identifyhome
+import os
+home = identifyhome.identify_home()
+if not home.endswith('/'): home += '/'
+
+app_root = os.path.dirname(os.path.realpath(__file__)) + '/../../'
+usage_file = home + 'disk-usage.txt'
+block_data_location = home + 'blocks/'
+contacts_location = home + 'contacts/'
+public_API_host_file = home + 'public-host.txt'
+private_API_host_file = home + 'private-host.txt'
+bootstrap_file_location = 'static-data/bootstrap-nodes.txt'
+data_nonce_file = home + 'block-nonces.dat'
+forward_keys_file = home + 'forward-keys.db'
+cached_storage = home + 'cachedstorage.dat'
+announce_cache = home + 'announcecache.dat'
+export_location = home + 'block-export/'
+upload_list = home + 'upload-list.json'
+config_file = home + 'config.json'
+daemon_mark_file = app_root + '/daemon-true.txt'
+lock_file = home + 'onionr.lock'
+
+site_cache = home + 'onionr-sites.txt'
+
+tor_hs_address_file = home + 'hs/hostname'
+
+run_check_file = home + '.runcheck'
+
+data_nonce_file = home + 'block-nonces.dat'
+
+keys_file = home + 'keys.txt'

src/httpapi/README.md

@@ -0,0 +1,13 @@
+# httpapi
+
+The httpapi contains collections of endpoints for the client and public API servers.
+
+## Files:
+
+configapi: manage onionr configuration from the client http api
+
+friendsapi: add, remove and list friends from the client http api
+
+miscpublicapi: misculanious onionr network interaction from the **public** httpapi, such as announcements, block fetching and uploading.
+
+profilesapi: work in progress in returning a profile page for an Onionr user

src/httpapi/__init__.py

@@ -0,0 +1,29 @@
+"""
+    Onionr - Private P2P Communication
+
+    This file registers plugin's flask blueprints for the client http server
+"""
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+import onionrplugins
+
+def load_plugin_blueprints(flaskapp, blueprint: str = 'flask_blueprint'):
+    """Iterate enabled plugins and load any http endpoints they have"""
+    for plugin in onionrplugins.get_enabled_plugins():
+        plugin = onionrplugins.get_plugin(plugin)
+        try:
+            flaskapp.register_blueprint(getattr(plugin, blueprint))
+        except AttributeError:
+            pass

src/httpapi/apiutils/__init__.py

@@ -0,0 +1,3 @@
+from . import shutdown, setbindip, getblockdata
+
+GetBlockData = getblockdata.GetBlockData

src/httpapi/apiutils/getblockdata.py

@@ -0,0 +1,35 @@
+import json
+from onionrblocks import onionrblockapi
+from onionrutils import bytesconverter, stringvalidators
+import onionrexceptions
+class GetBlockData:
+    def __init__(self, client_api_inst=None):
+        return
+    
+    def get_block_data(self, bHash, decrypt=False, raw=False, headerOnly=False):
+        if not stringvalidators.validate_hash(bHash): raise onionrexceptions.InvalidHexHash("block hash not valid hash format")
+        bl = onionrblockapi.Block(bHash)
+        if decrypt:
+            bl.decrypt()
+            if bl.isEncrypted and not bl.decrypted:
+                raise ValueError
+
+        if not raw:
+            if not headerOnly:
+                retData = {'meta':bl.bheader, 'metadata': bl.bmetadata, 'content': bl.bcontent}
+                for x in list(retData.keys()):
+                    try:
+                        retData[x] = retData[x].decode()
+                    except AttributeError:
+                        pass
+            else:
+                validSig = False
+                signer = bytesconverter.bytes_to_str(bl.signer)
+                if bl.isSigned() and stringvalidators.validate_pub_key(signer) and bl.isSigner(signer):
+                    validSig = True                    
+                bl.bheader['validSig'] = validSig
+                bl.bheader['meta'] = ''
+                retData = {'meta': bl.bheader, 'metadata': bl.bmetadata}
+            return json.dumps(retData)
+        else:
+            return bl.raw

src/httpapi/apiutils/setbindip.py

@@ -0,0 +1,42 @@
+import gevent
+from gevent import socket, sleep
+import secrets, random
+import config, logger
+import os
+
+# Hacky monkey patch so we can bind random localhosts without gevent trying to switch with an empty hub
+socket.getfqdn = lambda n: n
+
+def _get_acceptable_random_number()->int:
+    """Return a cryptographically random number in the inclusive range (1, 255)"""
+    number = 0
+    while number == 0:
+        number = secrets.randbelow(0xFF)
+    return number
+
+def set_bind_IP(filePath=''):
+    '''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)'''
+    if config.get('general.random_bind_ip', True):
+        hostOctets = []
+        # Build the random localhost address
+        for i in range(3):
+            hostOctets.append(str(_get_acceptable_random_number()))
+        hostOctets = ['127'] + hostOctets
+        # Convert the localhost address to a normal string address
+        data = '.'.join(hostOctets)
+        
+        # Try to bind IP. Some platforms like Mac block non normal 127.x.x.x
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            s.bind((data, 0))
+        except OSError:
+            # if mac/non-bindable, show warning and default to 127.0.0.1
+            logger.warn('Your platform appears to not support random local host addresses 127.x.x.x. Falling back to 127.0.0.1.')
+            data = '127.0.0.1'
+        s.close()
+    else:
+        data = '127.0.0.1'
+    if filePath != '':
+        with open(filePath, 'w') as bindFile:
+            bindFile.write(data)
+    return data

src/httpapi/apiutils/shutdown.py

@@ -0,0 +1,39 @@
+'''
+    Onionr - Private P2P Communication
+
+    Shutdown the node either hard or cleanly
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from flask import Blueprint, Response
+from onionrblocks import onionrblockapi
+import onionrexceptions
+from onionrutils import stringvalidators
+from coredb import daemonqueue
+shutdown_bp = Blueprint('shutdown', __name__)
+
+def shutdown(client_api_inst):
+    try:
+        client_api_inst.publicAPI.httpServer.stop()
+        client_api_inst.httpServer.stop()
+    except AttributeError:
+        pass
+    return Response("bye")
+
+@shutdown_bp.route('/shutdownclean')
+def shutdown_clean():
+    # good for calling from other clients
+    daemonqueue.daemon_queue_add('shutdown')
+    return Response("bye")

src/httpapi/configapi/__init__.py

@@ -0,0 +1,62 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file handles configuration setting and getting from the HTTP API
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import json
+from flask import Blueprint, request, Response, abort
+import config, onionrutils
+config.reload()
+
+config_BP = Blueprint('config_BP', __name__)
+
+@config_BP.route('/config/get')
+def get_all_config():
+    '''Simply return all configuration as JSON string'''
+    return Response(json.dumps(config.get_config(), indent=4, sort_keys=True))
+
+@config_BP.route('/config/get/<key>')
+def get_by_key(key):
+    '''Return a config setting by key'''
+    return Response(json.dumps(config.get(key)))
+
+@config_BP.route('/config/setall', methods=['POST'])
+def set_all_config():
+    '''Overwrite existing JSON config with new JSON string'''
+    try:
+        new_config = request.get_json(force=True)
+    except json.JSONDecodeError:
+        abort(400)
+    else:
+        config.set_config(new_config)
+        config.save()
+        return Response('success')
+
+@config_BP.route('/config/set/<key>', methods=['POST'])
+def set_by_key(key):
+    '''Overwrite/set only 1 config key'''
+    '''
+    {
+        'data': data
+    }
+    '''
+    try:
+        data = json.loads(onionrutils.OnionrUtils.bytesToStr(request.data))['data']
+    except (json.JSONDecodeError, KeyError):
+        abort(400)
+    config.set(key, data, True)
+    return Response('success')

src/httpapi/directconnections/__init__.py

@@ -0,0 +1,69 @@
+'''
+    Onionr - Private P2P Communication
+
+    Misc client API endpoints too small to need their own file and that need access to the client api inst
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import threading # For the client creation thread
+
+from flask import Response # For direct connection management HTTP endpoints
+from flask import Blueprint # To make the direct connection management blueprint in the webUI
+from flask import g # Mainly to access the shared toomanyobjs object
+import deadsimplekv
+
+import filepaths
+import onionrservices
+from onionrservices import pool
+
+def _get_communicator(g):
+    while True:
+        try:
+            return g.too_many.get_by_string("OnionrCommunicatorDaemon")
+        except KeyError:
+            pass
+
+class DirectConnectionManagement:
+    def __init__(self, client_api):
+        direct_conn_management_bp = Blueprint('direct_conn_management', __name__)
+        self.direct_conn_management_bp = direct_conn_management_bp
+
+        cache = deadsimplekv.DeadSimpleKV(filepaths.cached_storage)
+
+        @direct_conn_management_bp.route('/dc-client/isconnected/<pubkey>')
+        def is_connected(pubkey):
+            communicator = _get_communicator(g)
+            resp = ""
+            if pubkey in communicator.direct_connection_clients:
+                resp = communicator.direct_connection_clients[pubkey]
+            return Response(resp)
+        
+        @direct_conn_management_bp.route('/dc-client/connect/<pubkey>')
+        def make_new_connection(pubkey):
+            communicator = _get_communicator(g)
+            resp = "pending"
+            if pubkey in communicator.shared_state.get(pool.ServicePool).bootstrap_pending:
+                return Response(resp)
+                
+            if pubkey in communicator.direct_connection_clients:
+                resp = communicator.direct_connection_clients[pubkey]
+            else:
+                """Spawn a thread that will create the client and eventually add it to the
+                communicator.active_services 
+                """
+                threading.Thread(target=onionrservices.OnionrServices().create_client, 
+                                 args=[pubkey, communicator], daemon=True).start()
+
+            return Response(resp)

src/httpapi/fdsafehandler.py

@@ -0,0 +1,14 @@
+from gevent.pywsgi import WSGIServer, WSGIHandler
+from gevent import Timeout
+class FDSafeHandler(WSGIHandler):
+    '''Our WSGI handler. Doesn't do much non-default except timeouts'''
+    def handle(self):
+        self.timeout = Timeout(120, Exception)
+        self.timeout.start()
+        try:
+            WSGIHandler.handle(self)
+        except Timeout as ex:
+            if ex is self.timeout:
+                pass
+            else: 
+                raise

src/httpapi/friendsapi/__init__.py

@@ -0,0 +1,59 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file creates http endpoints for friend management
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import json
+from onionrusers import contactmanager
+from flask import Blueprint, Response, request, abort, redirect
+from coredb import keydb
+
+friends = Blueprint('friends', __name__)
+
+@friends.route('/friends/list')
+def list_friends():
+    pubkey_list = {}
+    friend_list = contactmanager.ContactManager.list_friends()
+    for friend in friend_list:
+        pubkey_list[friend.publicKey] = {'name': friend.get_info('name')}
+    return json.dumps(pubkey_list)
+
+@friends.route('/friends/add/<pubkey>', methods=['POST'])
+def add_friend(pubkey):
+    contactmanager.ContactManager(pubkey, saveUser=True).setTrust(1)
+    return redirect(request.referrer + '#' + request.form['token'])
+
+@friends.route('/friends/remove/<pubkey>', methods=['POST'])
+def remove_friend(pubkey):
+    contactmanager.ContactManager(pubkey).setTrust(0)
+    contactmanager.ContactManager(pubkey).delete_contact()
+    keydb.removekeys.remove_user(pubkey)
+    return redirect(request.referrer + '#' + request.form['token'])
+
+@friends.route('/friends/setinfo/<pubkey>/<key>', methods=['POST'])
+def set_info(pubkey, key):
+    data = request.form['data']
+    contactmanager.ContactManager(pubkey).set_info(key, data)
+    return redirect(request.referrer + '#' + request.form['token'])
+
+@friends.route('/friends/getinfo/<pubkey>/<key>')
+def get_info(pubkey, key):
+    retData = contactmanager.ContactManager(pubkey).get_info(key)
+    if retData is None:
+        abort(404)
+    else:
+        return retData

src/httpapi/insertblock.py

@@ -0,0 +1,73 @@
+'''
+    Onionr - Private P2P Communication
+
+    Create blocks with the client api server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import json, threading
+from flask import Blueprint, Response, request, g
+import onionrblocks
+from onionrcrypto import hashers
+from onionrutils import bytesconverter
+from onionrutils import mnemonickeys
+ib = Blueprint('insertblock', __name__)
+
+@ib.route('/insertblock', methods=['POST'])
+def client_api_insert_block():
+    encrypt = False
+    bData = request.get_json(force=True)
+    message = bData['message']
+    message_hash = bytesconverter.bytes_to_str(hashers.sha3_hash(message))
+
+    # Detect if message (block body) is not specified
+    if type(message) is None:
+        return 'failure due to unspecified message', 400
+
+    # Detect if block with same message is already being inserted
+    if message_hash in g.too_many.get_by_string("OnionrCommunicatorDaemon").generating_blocks:
+        return 'failure due to duplicate insert', 400
+    else:
+        g.too_many.get_by_string("OnionrCommunicatorDaemon").generating_blocks.append(message_hash)
+
+    subject = 'temp'
+    encryptType = ''
+    sign = True
+    meta = {}
+    to = ''
+    try:
+        if bData['encrypt']:
+            to = bData['to'].strip()
+            if "-" in to:
+                to = mnemonickeys.get_base32(to)
+            encrypt = True
+            encryptType = 'asym'
+    except KeyError:
+        pass
+    try:
+        if not bData['sign']:
+            sign = False
+    except KeyError:
+        pass
+    try:
+        bType = bData['type']
+    except KeyError:
+        bType = 'bin'
+    try:
+        meta = json.loads(bData['meta'])
+    except KeyError:
+        pass
+    threading.Thread(target=onionrblocks.insert, args=(message,), kwargs={'header': bType, 'encryptType': encryptType, 'sign':sign, 'asymPeer': to, 'meta': meta}).start()
+    return Response('success')

src/httpapi/miscclientapi/__init__.py

@@ -0,0 +1 @@
+from . import getblocks, staticfiles, endpoints, motd

src/httpapi/miscclientapi/endpoints.py

@@ -0,0 +1,138 @@
+'''
+    Onionr - Private P2P Communication
+
+    Misc client API endpoints too small to need their own file and that need access to the client api inst
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import os
+import subprocess
+
+from flask import Response, Blueprint, request, send_from_directory, abort
+import unpaddedbase32
+
+from httpapi import apiutils
+import onionrcrypto, config
+from netcontroller import NetController
+from serializeddata import SerializedData
+from onionrutils import mnemonickeys
+from onionrutils import bytesconverter
+from etc import onionrvalues
+from utils import reconstructhash
+from onionrcommands import restartonionr
+
+pub_key = onionrcrypto.pub_key.replace('=', '')
+
+SCRIPT_NAME = os.path.dirname(os.path.realpath(__file__)) + f'/../../../{onionrvalues.SCRIPT_NAME}'
+
+class PrivateEndpoints:
+    def __init__(self, client_api):
+        private_endpoints_bp = Blueprint('privateendpoints', __name__)
+        self.private_endpoints_bp = private_endpoints_bp
+
+        @private_endpoints_bp.route('/www/<path:path>', endpoint='www')
+        def wwwPublic(path):
+            if not config.get("www.private.run", True):
+                abort(403)
+            return send_from_directory(config.get('www.private.path', 'static-data/www/private/'), path)
+
+        @private_endpoints_bp.route('/hitcount')
+        def get_hit_count():
+            return Response(str(client_api.publicAPI.hitCount))
+
+        @private_endpoints_bp.route('/queueResponseAdd/<name>', methods=['post'])
+        def queueResponseAdd(name):
+            # Responses from the daemon. TODO: change to direct var access instead of http endpoint
+            client_api.queueResponse[name] = request.form['data']
+            return Response('success')
+        
+        @private_endpoints_bp.route('/queueResponse/<name>')
+        def queueResponse(name):
+            # Fetch a daemon queue response
+            resp = 'failure'
+            try:
+                resp = client_api.queueResponse[name]
+            except KeyError:
+                pass
+            else:
+                del client_api.queueResponse[name]
+            if resp == 'failure':
+                return resp, 404
+            else:
+                return resp
+            
+        @private_endpoints_bp.route('/ping')
+        def ping():
+            # Used to check if client api is working
+            return Response("pong!")
+
+        @private_endpoints_bp.route('/lastconnect')
+        def lastConnect():
+            return Response(str(client_api.publicAPI.lastRequest))
+
+        @private_endpoints_bp.route('/waitforshare/<name>', methods=['post'])
+        def waitforshare(name):
+            '''Used to prevent the **public** api from sharing blocks we just created'''
+            if not name.isalnum(): raise ValueError('block hash needs to be alpha numeric')
+            name = reconstructhash.reconstruct_hash(name)
+            if name in client_api.publicAPI.hideBlocks:
+                client_api.publicAPI.hideBlocks.remove(name)
+                return Response("removed")
+            else:
+                client_api.publicAPI.hideBlocks.append(name)
+                return Response("added")
+
+        @private_endpoints_bp.route('/shutdown')
+        def shutdown():
+            return apiutils.shutdown.shutdown(client_api)
+
+        @private_endpoints_bp.route('/restartclean')
+        def restart_clean():
+            subprocess.Popen([SCRIPT_NAME, 'restart'])
+            return Response("bye")
+        
+        @private_endpoints_bp.route('/getstats')
+        def getStats():
+            # returns node stats
+            while True:
+                try:    
+                    return Response(client_api._too_many.get(SerializedData).get_stats())
+                except AttributeError as e:
+                    pass
+        
+        @private_endpoints_bp.route('/getuptime')
+        def showUptime():
+            return Response(str(client_api.getUptime()))
+        
+        @private_endpoints_bp.route('/getActivePubkey')
+        def getActivePubkey():
+            return Response(pub_key)
+
+        @private_endpoints_bp.route('/getHumanReadable')
+        def getHumanReadableDefault():
+            return Response(mnemonickeys.get_human_readable_ID())
+
+        @private_endpoints_bp.route('/getHumanReadable/<name>')
+        def getHumanReadable(name):
+            name = unpaddedbase32.repad(bytesconverter.str_to_bytes(name))
+            return Response(mnemonickeys.get_human_readable_ID(name))
+        
+        @private_endpoints_bp.route('/getBase32FromHumanReadable/<words>')
+        def get_base32_from_human_readable(words):
+            return Response(bytesconverter.bytes_to_str(mnemonickeys.get_base32(words)))
+
+        @private_endpoints_bp.route('/gettorsocks')
+        def get_tor_socks():
+            return Response(str(client_api._too_many.get(NetController).socksPort))

src/httpapi/miscclientapi/getblocks.py

@@ -0,0 +1,65 @@
+'''
+    Onionr - Private P2P Communication
+
+    Create blocks with the client api server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from flask import Blueprint, Response, abort
+from onionrblocks import onionrblockapi
+from .. import apiutils
+from onionrutils import stringvalidators
+from coredb import blockmetadb
+
+client_get_block = apiutils.GetBlockData()
+
+client_get_blocks = Blueprint('miscclient', __name__)
+
+@client_get_blocks.route('/getblocksbytype/<name>')
+def get_blocks_by_type_endpoint(name):
+    blocks = blockmetadb.get_blocks_by_type(name)
+    return Response(','.join(blocks))
+
+@client_get_blocks.route('/getblockbody/<name>')
+def getBlockBodyData(name):
+    resp = ''
+    if stringvalidators.validate_hash(name):
+        try:
+            resp = onionrblockapi.Block(name, decrypt=True).bcontent
+        except TypeError:
+            pass
+    else:
+        abort(404)
+    return Response(resp)
+
+@client_get_blocks.route('/getblockdata/<name>')
+def getData(name):
+    resp = ""
+    if stringvalidators.validate_hash(name):
+        if name in blockmetadb.get_block_list():
+            try:
+                resp = client_get_block.get_block_data(name, decrypt=True)
+            except ValueError:
+                pass
+        else:
+            abort(404)
+    else:
+        abort(404)
+    return Response(resp)
+
+@client_get_blocks.route('/getblockheader/<name>')
+def getBlockHeader(name):
+    resp = client_get_block.get_block_data(name, decrypt=True, headerOnly=True)
+    return Response(resp)

src/httpapi/miscclientapi/motd/__init__.py

@@ -0,0 +1,27 @@
+from flask import Blueprint
+from flask import Response
+import unpaddedbase32
+
+from coredb import blockmetadb
+import onionrblocks
+from etc import onionrvalues
+import config
+from onionrutils import bytesconverter
+
+bp = Blueprint('motd', __name__)
+
+signer = config.get("motd.motd_key", onionrvalues.MOTD_SIGN_KEY)
+
+@bp.route('/getmotd')
+def get_motd()->Response:
+    motds = blockmetadb.get_blocks_by_type("motd")
+    newest_time = 0
+    message = "No MOTD currently present."
+    for x in motds:
+        bl = onionrblocks.onionrblockapi.Block(x)
+        if not bl.verifySig() or bl.signer != bytesconverter.bytes_to_str(unpaddedbase32.repad(bytesconverter.str_to_bytes(signer))): continue
+        if not bl.isSigner(signer): continue
+        if bl.claimedTime > newest_time:
+            newest_time = bl.claimedTime
+            message = bl.bcontent
+    return Response(message, headers={"Content-Type": "text/plain"})

src/httpapi/miscclientapi/staticfiles.py

@@ -0,0 +1,92 @@
+'''
+    Onionr - Private P2P Communication
+
+    Register static file routes
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import os
+import mimetypes
+from flask import Blueprint, send_from_directory
+
+# Was having some mime type issues on windows, this appeared to fix it.
+# we have no-sniff set, so if the mime types are invalid sripts can't load.
+mimetypes.add_type('application/javascript', '.js')
+mimetypes.add_type('text/css', '.css')
+
+static_files_bp = Blueprint('staticfiles', __name__)
+
+root = os.path.dirname(os.path.realpath(__file__)) + '/../../../static-data/www/' # should be set to onionr install directory from onionr startup
+
+@static_files_bp.route('/onboarding/', endpoint='onboardingIndex')
+def onboard():
+    return send_from_directory(f'{root}onboarding/', "index.html")
+
+@static_files_bp.route('/onboarding/<path:path>', endpoint='onboarding')
+def onboard_files(path):
+    return send_from_directory(f'{root}onboarding/', path)
+
+@static_files_bp.route('/chat/', endpoint='chatIndex')
+def chat_index():
+    return send_from_directory(root + 'chat/', "index.html")
+
+@static_files_bp.route('/chat/<path:path>', endpoint='chat')
+def load_chat(path):
+    return send_from_directory(root + 'chat/', path)
+
+@static_files_bp.route('/board/', endpoint='board')
+def loadBoard():
+    return send_from_directory(root + 'board/', "index.html")
+
+@static_files_bp.route('/mail/<path:path>', endpoint='mail')
+def loadMail(path):
+    return send_from_directory(root + 'mail/', path)
+
+@static_files_bp.route('/mail/', endpoint='mailindex')
+def loadMailIndex():
+    return send_from_directory(root + 'mail/', 'index.html')
+
+@static_files_bp.route('/friends/<path:path>', endpoint='friends')
+def loadContacts(path):
+    return send_from_directory(root + 'friends/', path)
+
+@static_files_bp.route('/friends/', endpoint='friendsindex')
+def loadContacts():
+    return send_from_directory(root + 'friends/', 'index.html')
+
+@static_files_bp.route('/profiles/<path:path>', endpoint='profiles')
+def loadContacts(path):
+    return send_from_directory(root + 'profiles/', path)
+
+@static_files_bp.route('/profiles/', endpoint='profilesindex')
+def loadContacts():
+    return send_from_directory(root + 'profiles/', 'index.html')
+
+@static_files_bp.route('/board/<path:path>', endpoint='boardContent')
+def boardContent(path):
+    return send_from_directory(root + 'board/', path)
+
+@static_files_bp.route('/shared/<path:path>', endpoint='sharedContent')
+def sharedContent(path):
+    return send_from_directory(root + 'shared/', path)
+
+@static_files_bp.route('/', endpoint='onionrhome')
+def hello():
+    # ui home
+    return send_from_directory(root + 'private/', 'index.html')
+
+@static_files_bp.route('/private/<path:path>', endpoint='homedata')
+def homedata(path):
+    return send_from_directory(root + 'private/', path)

src/httpapi/miscpublicapi/__init__.py

@@ -0,0 +1,6 @@
+from . import announce, upload, getblocks, endpoints
+
+announce = announce.handle_announce # endpoint handler for accepting peer announcements
+upload = upload.accept_upload # endpoint handler for accepting public uploads
+public_block_list = getblocks.get_public_block_list # endpoint handler for getting block lists
+public_get_block_data = getblocks.get_block_data # endpoint handler for responding to peers requests for block data

src/httpapi/miscpublicapi/announce.py

@@ -0,0 +1,76 @@
+'''
+    Onionr - Private P2P Communication
+
+    Handle announcements to the public API server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import base64
+from flask import Response, g
+import deadsimplekv
+import logger
+from etc import onionrvalues
+from onionrutils import stringvalidators, bytesconverter
+from utils import gettransports
+import onionrcrypto as crypto, filepaths
+from communicator import OnionrCommunicatorDaemon
+def handle_announce(request):
+    '''
+    accept announcement posts, validating POW
+    clientAPI should be an instance of the clientAPI server running, request is a instance of a flask request
+    '''
+    resp = 'failure'
+    powHash = ''
+    randomData = ''
+    newNode = ''
+
+    try:
+        newNode = request.form['node'].encode()
+    except KeyError:
+        logger.warn('No node specified for upload')
+        pass
+    else:
+        try:
+            randomData = request.form['random']
+            randomData = base64.b64decode(randomData)
+        except KeyError:
+            logger.warn('No random data specified for upload')
+        else:
+            nodes = newNode + bytesconverter.str_to_bytes(gettransports.get()[0])
+            nodes = crypto.hashers.blake2b_hash(nodes)
+            powHash = crypto.hashers.blake2b_hash(randomData + nodes)
+            try:
+                powHash = powHash.decode()
+            except AttributeError:
+                pass
+            if powHash.startswith('0' * onionrvalues.ANNOUNCE_POW):
+                newNode = bytesconverter.bytes_to_str(newNode)
+                announce_queue = deadsimplekv.DeadSimpleKV(filepaths.announce_cache)
+                announce_queue_list = announce_queue.get('new_peers')
+                if announce_queue_list is None:
+                    announce_queue_list = []
+
+                if stringvalidators.validate_transport(newNode) and not newNode in announce_queue_list:
+                    #clientAPI.onionrInst.communicatorInst.newPeers.append(newNode)
+                    g.shared_state.get(OnionrCommunicatorDaemon).newPeers.append(newNode)
+                    announce_queue.put('new_peers', announce_queue_list.append(newNode))
+                    announce_queue.flush()
+                    resp = 'Success'
+            else:
+                logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)
+    resp = Response(resp)
+    if resp == 'failure':
+        return resp, 406
+    return resp

src/httpapi/miscpublicapi/endpoints.py

@@ -0,0 +1,81 @@
+'''
+    Onionr - Private P2P Communication
+
+    Misc public API endpoints too small to need their own file and that need access to the public api inst
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from flask import Response, Blueprint, request, send_from_directory, abort, g
+from . import getblocks, upload, announce
+from coredb import keydb
+import config
+class PublicEndpoints:
+    def __init__(self, public_api):
+
+        public_endpoints_bp = Blueprint('publicendpoints', __name__)
+        self.public_endpoints_bp = public_endpoints_bp
+
+        @public_endpoints_bp.route('/')
+        def banner():
+            # Display a bit of information to people who visit a node address in their browser
+            try:
+                with open('../static-data/index.html', 'r') as html:
+                    resp = Response(html.read(), mimetype='text/html')
+            except FileNotFoundError:
+                resp = Response("")
+            return resp
+
+        @public_endpoints_bp.route('/getblocklist')
+        def get_block_list():
+            '''Get a list of blocks, optionally filtered by epoch time stamp, excluding those hidden'''
+            return getblocks.get_public_block_list(public_api, request)
+
+        @public_endpoints_bp.route('/getdata/<name>')
+        def get_block_data(name):
+            # Share data for a block if we have it and it isn't hidden
+            return getblocks.get_block_data(public_api, name)
+
+        @public_endpoints_bp.route('/www/<path:path>')
+        def www_public(path):
+            # A way to share files directly over your .onion
+            if not config.get("www.public.run", True):
+                abort(403)
+            return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
+
+        @public_endpoints_bp.route('/ping')
+        def ping():
+            # Endpoint to test if nodes are up
+            return Response("pong!")
+
+        @public_endpoints_bp.route('/pex')
+        def peer_exchange():
+            response = ','.join(keydb.listkeys.list_adders(recent=3600))
+            if len(response) == 0:
+                response = ''
+            return Response(response)
+
+        @public_endpoints_bp.route('/announce', methods=['post'])
+        def accept_announce():
+            '''Accept announcements with pow token to prevent spam'''
+            g.shared_state = public_api._too_many
+            resp = announce.handle_announce(request)
+            return resp
+
+        @public_endpoints_bp.route('/upload', methods=['post'])
+        def upload_endpoint():
+            '''Accept file uploads. In the future this will be done more often than on creation 
+            to speed up block sync
+            '''
+            return upload.accept_upload(request)

src/httpapi/miscpublicapi/getblocks.py

@@ -0,0 +1,58 @@
+'''
+    Onionr - Private P2P Communication
+
+    Public endpoints to get block data and lists
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from flask import Response, abort
+import config
+from onionrutils import bytesconverter, stringvalidators
+from coredb import blockmetadb
+from utils import reconstructhash
+from .. import apiutils
+def get_public_block_list(publicAPI, request):
+    # Provide a list of our blocks, with a date offset
+    dateAdjust = request.args.get('date')
+    bList = blockmetadb.get_block_list(dateRec=dateAdjust)
+    share_list = ''
+    if config.get('general.hide_created_blocks', True):
+        for b in publicAPI.hideBlocks:
+            if b in bList:
+                # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
+                bList.remove(b)
+    for b in bList:
+        share_list += '%s\n' % (reconstructhash.deconstruct_hash(b),)
+    return Response(share_list)
+
+def get_block_data(publicAPI, data):
+    '''data is the block hash in hex'''
+    resp = ''
+    if stringvalidators.validate_hash(data):
+        if not config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks:
+            if data in blockmetadb.get_block_list():
+                block = apiutils.GetBlockData().get_block_data(data, raw=True)
+                try:
+                    block = block.encode() # Encode in case data is binary
+                except AttributeError:
+                    if len(block) == 0:
+                        abort(404)
+                block = bytesconverter.str_to_bytes(block)
+                resp = block
+    if len(resp) == 0:
+        abort(404)
+        resp = ""
+    # Has to be octet stream, otherwise binary data fails hash check
+    return Response(resp, mimetype='application/octet-stream')

src/httpapi/miscpublicapi/upload.py

@@ -0,0 +1,49 @@
+'''
+    Onionr - Private P2P Communication
+
+    Accept block uploads to the public API server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys
+from flask import Response, abort
+
+from onionrblocks import blockimporter
+import onionrexceptions, logger
+
+def accept_upload(request):
+    resp = 'failure'
+    data = request.get_data()
+    if sys.getsizeof(data) < 100000000:
+        try:
+            if blockimporter.importBlockFromData(data):
+                resp = 'success'
+            else:
+                resp = 'failure'
+                logger.warn('Error encountered importing uploaded block')
+        except onionrexceptions.BlacklistedBlock:
+            logger.debug('uploaded block is blacklisted')
+            resp = 'failure'
+        except onionrexceptions.InvalidProof:
+            resp = 'proof'
+        except onionrexceptions.DataExists:
+            resp = 'exists'
+    if resp == 'failure':
+        abort(400)
+    elif resp == 'proof':
+        resp = Response(resp, 400)
+    else:
+        resp = Response(resp)
+    return resp

src/httpapi/onionrsitesapi/__init__.py

@@ -0,0 +1,95 @@
+"""
+    Onionr - Private P2P Communication
+
+    view and interact with onionr sites
+"""
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+import base64
+import binascii
+import mimetypes
+
+import unpaddedbase32
+
+from flask import Blueprint, Response, request, abort
+
+from onionrblocks import onionrblockapi
+import onionrexceptions
+from onionrutils import stringvalidators
+from utils import safezip
+from onionrutils import mnemonickeys
+from . import sitefiles
+
+site_api = Blueprint('siteapi', __name__)
+
+@site_api.route('/site/<name>/', endpoint='site')
+def site(name: str)->Response:
+    """Accept a site 'name', if pubkey then show multi-page site, if hash show single page site"""
+    resp: str = 'Not Found'
+    mime_type = 'text/html'
+
+    # If necessary convert the name to base32 from mnemonic
+    if mnemonickeys.DELIMITER in name:
+        name = mnemonickeys.get_base32(name)
+
+    # Now make sure the key is regardless a valid base32 format ed25519 key (readding padding if necessary)
+    if stringvalidators.validate_pub_key(name):
+        name = unpaddedbase32.repad(name)
+        resp = sitefiles.get_file(name, 'index.html')
+
+    elif stringvalidators.validate_hash(name):
+        try:
+            resp = onionrblockapi.Block(name).bcontent
+        except onionrexceptions.NoDataAvailable:
+            abort(404)
+        except TypeError:
+            pass
+        try:
+            resp = base64.b64decode(resp)
+        except binascii.Error:
+            pass
+    if resp == 'Not Found' or not resp:
+        abort(404)
+    return Response(resp)
+
+@site_api.route('/site/<name>/<path:file>', endpoint='siteFile')
+def site_file(name: str, file: str)->Response:
+    """Accept a site 'name', if pubkey then show multi-page site, if hash show single page site"""
+    resp: str = 'Not Found'
+    mime_type = mimetypes.MimeTypes().guess_type(file)[0]
+
+    # If necessary convert the name to base32 from mnemonic
+    if mnemonickeys.DELIMITER in name:
+        name = mnemonickeys.get_base32(name)
+
+    # Now make sure the key is regardless a valid base32 format ed25519 key (readding padding if necessary)
+    if stringvalidators.validate_pub_key(name):
+        name = unpaddedbase32.repad(name)
+        resp = sitefiles.get_file(name, file)
+
+    elif stringvalidators.validate_hash(name):
+        try:
+            resp = onionrblockapi.Block(name).bcontent
+        except onionrexceptions.NoDataAvailable:
+            abort(404)
+        except TypeError:
+            pass
+        try:
+            resp = base64.b64decode(resp)
+        except binascii.Error:
+            pass
+    if resp == 'Not Found' or not resp:
+        abort(404)
+    return Response(resp, mimetype=mime_type)

src/httpapi/onionrsitesapi/findsite.py

@@ -0,0 +1,49 @@
+"""
+    Onionr - Private P2P Communication
+
+    view and interact with onionr sites
+"""
+
+from typing import Union
+
+import onionrexceptions
+from onionrutils import mnemonickeys
+from onionrutils import stringvalidators
+from coredb import blockmetadb
+from onionrblocks.onionrblockapi import Block
+from onionrtypes import BlockHash
+
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+
+def find_site(user_id: str) -> Union[BlockHash, None]:
+    """Returns block hash str for latest block for a site by a given user id"""
+    # If mnemonic delim in key, convert to base32 version
+    if mnemonickeys.DELIMITER in user_id:
+        user_id = mnemonickeys.get_base32(user_id)
+
+    if not stringvalidators.validate_pub_key(user_id):
+        raise onionrexceptions.InvalidPubkey
+
+    found_site = None
+    sites = blockmetadb.get_blocks_by_type('zsite')
+
+    # Find site by searching all site blocks. eww O(N) ☹️, TODO: event based
+    for site in sites:
+        site = Block(site)
+        if site.isSigner(user_id) and site.verifySig():
+            found_site = site.hash
+    return found_site

src/httpapi/onionrsitesapi/sitefiles.py

@@ -0,0 +1,72 @@
+"""
+    Onionr - Private P2P Communication
+
+    Read onionr site files
+"""
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+from typing import Union, Tuple
+import tarfile
+import io
+import os
+
+import unpaddedbase32
+
+from coredb import blockmetadb
+from onionrblocks import onionrblockapi
+from onionrblocks import insert
+
+# Import types. Just for type hiting
+from onionrtypes import UserID, DeterministicKeyPassphrase, BlockHash
+
+from onionrcrypto import generate_deterministic
+
+def find_site_gzip(user_id: str)->tarfile.TarFile:
+    """Return verified site tar object"""
+    sites = blockmetadb.get_blocks_by_type('osite')
+    user_site = None
+    user_id = unpaddedbase32.repad(user_id)
+    for site in sites:
+        block = onionrblockapi.Block(site)
+        if block.isSigner(user_id):
+            user_site = block
+    if not user_site is None:
+        return tarfile.open(fileobj=io.BytesIO(user_site.bcontent), mode='r')
+    return None
+
+def get_file(user_id, file)->Union[bytes, None]:
+    """Get a site file content"""
+    ret_data = ""
+    site = find_site_gzip(user_id)
+    if site is None: return None
+    for t_file in site.getmembers():
+        if t_file.name.replace('./', '') == file:
+            return site.extractfile(t_file)
+    return None
+
+def create_site(admin_pass: DeterministicKeyPassphrase, directory:str='.')->Tuple[UserID, BlockHash]:
+    public_key, private_key = generate_deterministic(admin_pass)
+
+    raw_tar = io.BytesIO()
+
+    tar = tarfile.open(mode='x:gz', fileobj=raw_tar)
+    tar.add(directory)
+    tar.close()
+
+    raw_tar.seek(0)
+
+    block_hash = insert(raw_tar.read(), header='osite', signing_key=private_key, sign=True)
+
+    return (public_key, block_hash)

src/httpapi/profilesapi/__init__.py

@@ -0,0 +1,27 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file creates http endpoints for user profile pages
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from flask import Blueprint, Response, request, abort
+from . import profiles
+
+profile_BP = Blueprint('profile_BP', __name__)
+
+@profile_BP.route('/profile/get/<pubkey>', endpoint='profiles')
+def get_profile_page(pubkey):
+    return Response(pubkey)

src/httpapi/profilesapi/profiles.py

@@ -0,0 +1,2 @@
+def get_latest_user_profile(pubkey):
+    return ''

src/httpapi/security/__init__.py

@@ -0,0 +1 @@
+from . import client, public

src/httpapi/security/client.py

@@ -0,0 +1,69 @@
+'''
+    Onionr - Private P2P Communication
+
+    Process incoming requests to the client api server to validate they are legitimate
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import hmac
+from flask import Blueprint, request, abort, g
+from onionrservices import httpheaders
+from . import pluginwhitelist
+
+# Be extremely mindful of this. These are endpoints available without a password
+whitelist_endpoints = ['www', 'staticfiles.homedata', 'staticfiles.sharedContent', 
+'staticfiles.friends', 'staticfiles.friendsindex', 'siteapi.site', 'siteapi.siteFile', 'staticfiles.onionrhome',
+'themes.getTheme', 'staticfiles.onboarding', 'staticfiles.onboardingIndex']
+
+class ClientAPISecurity:
+    def __init__(self, client_api):
+        client_api_security_bp = Blueprint('clientapisecurity', __name__)
+        self.client_api_security_bp = client_api_security_bp
+        self.client_api = client_api
+        pluginwhitelist.load_plugin_security_whitelist_endpoints(whitelist_endpoints)
+
+        @client_api_security_bp.before_app_request
+        def validate_request():
+            '''Validate request has set password and is the correct hostname'''
+            # For the purpose of preventing DNS rebinding attacks
+            if request.host != '%s:%s' % (client_api.host, client_api.bindPort):
+                abort(403)
+
+            # Add shared objects
+            try:
+                g.too_many = self.client_api._too_many
+            except KeyError:
+                g.too_many = None
+
+            if request.endpoint in whitelist_endpoints:
+                return
+            if request.path.startswith('/site/'): return
+            try:
+                if not hmac.compare_digest(request.headers['token'], client_api.clientToken):
+                    if not hmac.compare_digest(request.form['token'], client_api.clientToken):
+                        abort(403)
+            except KeyError:
+                if not hmac.compare_digest(request.form['token'], client_api.clientToken):
+                    abort(403)
+
+        @client_api_security_bp.after_app_request
+        def after_req(resp):
+            # Security headers
+            resp = httpheaders.set_default_onionr_http_headers(resp)
+            if request.endpoint in ('siteapi.site', 'siteapi.siteFile'):
+                resp.headers['Content-Security-Policy'] = "default-src 'none'; style-src 'self' data: 'unsafe-inline'; img-src 'self' data:; media-src 'self' data:"
+            else:
+                resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; media-src 'none'; frame-src 'none'; font-src 'self'; connect-src 'self'"
+            return resp

src/httpapi/security/pluginwhitelist.py

@@ -0,0 +1,32 @@
+"""
+    Onionr - Private P2P Communication
+
+    Load web UI client endpoints into the whitelist from plugins
+"""
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+import onionrplugins
+def load_plugin_security_whitelist_endpoints(whitelist: list):
+    """Accept a list reference of whitelist endpoints from security/client.py and
+    append plugin's specified endpoints to them by attribute"""
+    for plugin in onionrplugins.get_enabled_plugins():
+        try:
+            plugin = onionrplugins.get_plugin(plugin)
+        except FileNotFoundError:
+            continue
+        try:
+            whitelist.extend(getattr(plugin, "security_whitelist"))
+        except AttributeError:
+            pass

src/httpapi/security/public.py

@@ -0,0 +1,60 @@
+'''
+    Onionr - Private P2P Communication
+
+    Process incoming requests to the public api server for certain attacks
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from flask import Blueprint, request, abort, g
+from onionrservices import httpheaders
+from onionrutils import epoch
+from utils import gettransports
+class PublicAPISecurity:
+    def __init__(self, public_api):
+        public_api_security_bp = Blueprint('publicapisecurity', __name__)
+        self.public_api_security_bp = public_api_security_bp
+
+        @public_api_security_bp.before_app_request
+        def validate_request():
+            '''Validate request has the correct hostname'''
+            # If high security level, deny requests to public (HS should be disabled anyway for Tor, but might not be for I2P)
+            transports = gettransports.get()
+            if public_api.config.get('general.security_level', default=1) > 0:
+                abort(403)
+            if request.host not in transports:
+                # Disallow connection if wrong HTTP hostname, in order to prevent DNS rebinding attacks
+                abort(403)
+            public_api.hitCount += 1 # raise hit count for valid requests
+            try:
+                if 'onionr' in request.headers['User-Agent'].lower():
+                    g.is_onionr_client = True
+                else:
+                    g.is_onionr_client = False
+            except KeyError:
+                g.is_onionr_client = False
+
+        @public_api_security_bp.after_app_request
+        def send_headers(resp):
+            '''Send api, access control headers'''
+            resp = httpheaders.set_default_onionr_http_headers(resp)
+            # Network API version
+            resp.headers['X-API'] = public_api.API_VERSION
+            # Delete some HTTP headers for Onionr user agents
+            NON_NETWORK_HEADERS = ('Content-Security-Policy', 'X-Frame-Options',
+            'X-Content-Type-Options', 'Feature-Policy', 'Clear-Site-Data', 'Referrer-Policy')
+            if g.is_onionr_client:
+                for header in NON_NETWORK_HEADERS: del resp.headers[header]
+            public_api.lastRequest = epoch.get_rounded_epoch(roundS=5)
+            return resp

src/httpapi/themeapi/__init__.py

@@ -0,0 +1,46 @@
+"""
+    Onionr - Private P2P Communication
+
+    API to get current CSS theme for the client web UI
+"""
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+from flask import Blueprint, Response
+
+import config
+from utils import readstatic
+
+theme_blueprint = Blueprint('themes', __name__)
+
+LIGHT_THEME_FILES = ['bulma-light.min.css', 'styles-light.css']
+DARK_THEME_FILES = ['bulma-dark.min.css', 'styles-dark.css']
+
+def _load_from_files(file_list: list)->str:
+    """Loads multiple static dir files and returns them in combined string format (non-binary)"""
+    combo_data = ''
+    for f in file_list:
+        combo_data += readstatic.read_static('www/shared/main/themes/' + f)
+    return combo_data
+
+@theme_blueprint.route('/gettheme', endpoint='getTheme')
+def get_theme_file()->Response:
+    """Returns the css theme data"""
+    css: str
+    theme = config.get('ui.theme', 'dark').lower()
+    if theme == 'dark':
+        css = _load_from_files(DARK_THEME_FILES)
+    elif theme == 'light':
+        css = _load_from_files(LIGHT_THEME_FILES)
+    return Response(css, mimetype='text/css')

src/keymanager.py

@@ -0,0 +1,78 @@
+'''
+    Onionr - Private P2P Communication
+
+    Load, save, and delete the user's public key pairs (does not handle peer keys)
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from onionrutils import bytesconverter
+from onionrcrypto import generate
+import os
+import filepaths
+
+class KeyManager:
+    def __init__(self):
+        self.keyFile = filepaths.keys_file
+
+    def addKey(self, pubKey=None, privKey=None):
+        '''Add a new key pair, either specified or none to generate a new pair automatically'''
+        if type(pubKey) is type(None) and type(privKey) is type(None):
+            pubKey, privKey = generate.generate_pub_key()
+        pubKey = bytesconverter.bytes_to_str(pubKey)
+        privKey = bytesconverter.bytes_to_str(privKey)
+        try:
+            if pubKey in self.getPubkeyList():
+                raise ValueError('Pubkey already in list: %s' % (pubKey,))
+        except FileNotFoundError:
+            pass
+
+        with open(self.keyFile, "a") as keyFile:
+            keyFile.write(pubKey + ',' + privKey + '\n')
+        return (pubKey, privKey)
+
+    def removeKey(self, pubKey):
+        '''Remove a key pair by pubkey'''
+        keyList = self.getPubkeyList()
+        keyData = ''
+        try:
+            keyList.remove(pubKey)
+        except ValueError:
+            return False
+        else:
+            keyData = ','.join(keyList)
+            with open(self.keyFile, "w") as keyFile:
+                keyFile.write(keyData)
+
+    def getPubkeyList(self):
+        '''Return a list of the user's keys'''
+        keyList = []
+        try:
+            with open(self.keyFile, "r") as keyFile:
+                keyData = keyFile.read()
+        except FileNotFoundError:
+            keyData = ''
+        keyData = keyData.split('\n')
+        for pair in keyData:
+            if len(pair) > 0: keyList.append(pair.split(',')[0])
+        return keyList
+    
+    def getPrivkey(self, pubKey):
+        privKey = None
+        with open(self.keyFile, "r") as keyFile:
+            keyData = keyFile.read()
+        for pair in keyData.split('\n'):
+            if pubKey in pair or pubKey.replace('=', '') in pair:
+                privKey = pair.split(',')[1]
+        return privKey

src/logger/__init__.py

@@ -0,0 +1,71 @@
+'''
+    Onionr - Private P2P Communication
+
+    This file handles all operations involving logging
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import sys, traceback
+
+from . import colors, readline, log, raw, confirm, colors, settings
+colors = colors.Colors
+readline = readline.readline
+log = log.log
+raw = raw.raw
+confirm = confirm.confirm
+
+# debug: when there is info that could be useful for debugging purposes only
+def debug(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_DEBUG):
+    if settings.get_level() <= level:
+        log('/', data, timestamp = timestamp, prompt = prompt, terminal = terminal)
+    if not error is None:
+        debug('Error: ' + str(error) + parse_error())
+
+# info: when there is something to notify the user of, such as the success of a process
+def info(data: str, timestamp = False, prompt = True, terminal = False, level = settings.LEVEL_INFO):
+    if settings.get_level() <= level:
+        log('+', data, colors.fg.green, timestamp = timestamp, prompt = prompt, terminal = terminal)
+
+# warn: when there is a potential for something bad to happen
+def warn(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_WARN):
+    if not error is None:
+        debug('Error: ' + str(error) + parse_error())
+    if settings.get_level() <= level:
+        log('!', data, colors.fg.orange, timestamp = timestamp, prompt = prompt, terminal = terminal)
+
+# error: when only one function, module, or process of the program encountered a problem and must stop
+def error(data: str, error = None, timestamp = True, prompt = True, terminal = False, level = settings.LEVEL_ERROR):
+    if settings.get_level() <= level:
+        log('-', data, colors.fg.red, timestamp = timestamp, fd = sys.stderr, prompt = prompt, terminal = terminal)
+    if not error is None:
+        debug('Error: ' + str(error) + parse_error())
+
+# fatal: when the something so bad has happened that the program must stop
+def fatal(data: str, error = None, timestamp=True, prompt = True, terminal = False, level = settings.LEVEL_FATAL):
+    if not error is None:
+        debug('Error: ' + str(error) + parse_error(), terminal = terminal)
+    if settings.get_level() <= level:
+        log('#', data, colors.bg.red + colors.fg.green + colors.bold, timestamp = timestamp, fd = sys.stderr, prompt = prompt, terminal = terminal)
+
+# returns a formatted error message
+def parse_error():
+    details = traceback.extract_tb(sys.exc_info()[2])
+    output = ''
+
+    for line in details:
+        output += '\n    ... module %s in  %s:%i' % (line[2], line[0], line[1])
+
+    return output

src/logger/colors.py

@@ -0,0 +1,60 @@
+'''
+    Onionr - Private P2P Communication
+
+    class to access ANSI control codes
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import re
+class Colors:
+    '''
+        This class allows you to set the color if ANSI codes are supported
+    '''
+    reset='\033[0m'
+    bold='\033[01m'
+    disable='\033[02m'
+    underline='\033[04m'
+    reverse='\033[07m'
+    strikethrough='\033[09m'
+    invisible='\033[08m'
+    italics='\033[3m'
+    class fg:
+        black='\033[30m'
+        red='\033[31m'
+        green='\033[32m'
+        orange='\033[33m'
+        blue='\033[34m'
+        purple='\033[35m'
+        cyan='\033[36m'
+        lightgrey='\033[37m'
+        darkgrey='\033[90m'
+        lightred='\033[91m'
+        lightgreen='\033[92m'
+        yellow='\033[93m'
+        lightblue='\033[94m'
+        pink='\033[95m'
+        lightcyan='\033[96m'
+    class bg:
+        black='\033[40m'
+        red='\033[41m'
+        green='\033[42m'
+        orange='\033[43m'
+        blue='\033[44m'
+        purple='\033[45m'
+        cyan='\033[46m'
+        lightgrey='\033[47m'
+    @staticmethod
+    def filter(data):
+        return re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]').sub('', str(data))

src/logger/confirm.py

@@ -0,0 +1,54 @@
+'''
+    Onionr - Private P2P Communication
+
+    confirm y/n cli prompt
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys
+from . import colors, settings
+colors = colors.Colors
+def confirm(default = 'y', message = 'Are you sure %s? '):
+    '''
+        Displays an "Are you sure" message, returns True for Y and False for N
+        message: The confirmation message, use %s for (y/n)
+        default: which to prefer-- y or n
+    '''
+
+    color = colors.fg.green + colors.bold
+
+    default = default.lower()
+    confirm = colors.bold
+    if default.startswith('y'):
+        confirm += '(Y/n)'
+    else:
+        confirm += '(y/N)'
+    confirm += colors.reset + color
+
+    output = colors.reset + str(color) + '... ' + colors.reset + str(message) + colors.reset
+
+    if not get_settings() & settings.USE_ANSI:
+        output = colors.filter(output)
+
+    sys.stdout.write(output.replace('%s', confirm))
+
+    inp = input().lower()
+
+    if 'y' in inp:
+        return True
+    if 'n' in inp:
+        return False
+    else:
+        return default == 'y'

src/logger/log.py

@@ -0,0 +1,38 @@
+'''
+    Onionr - Private P2P Communication
+
+    god log function
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys, time
+from . import colors, raw, settings
+colors = colors.Colors
+def log(prefix, data, color = '', timestamp=True, fd = sys.stdout, prompt = True, terminal = False):
+    '''
+        Logs the data
+        prefix : The prefix to the output
+        data   : The actual data to output
+        color  : The color to output before the data
+    '''
+    curTime = ''
+    if timestamp:
+        curTime = time.strftime("%m-%d %H:%M:%S") + ' '
+
+    output = colors.reset + str(color) + ('[' + colors.bold + str(prefix) + colors.reset + str(color) + '] ' if prompt is True else '') + curTime + str(data) + colors.reset
+    if not settings.get_settings() & settings.USE_ANSI:
+        output = colors.filter(output)
+
+    raw.raw(output, fd = fd, terminal = terminal)

src/logger/raw.py

@@ -0,0 +1,46 @@
+'''
+    Onionr - Private P2P Communication
+
+    Output raw data to file or terminal
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys, os
+from . import settings, colors
+colors = colors.Colors
+def raw(data, fd = sys.stdout, terminal = False):
+    '''
+        Outputs raw data to console without formatting
+    '''
+
+    if terminal and (settings.get_settings() & settings.OUTPUT_TO_CONSOLE):
+        try:
+            ts = fd.write('%s\n' % data)
+        except OSError:
+            pass
+    if settings.get_settings() & settings.OUTPUT_TO_FILE:
+        fdata = ''
+        try:
+            with open(settings._outputfile, 'r') as file:
+                fdata = file.read()
+        except FileNotFoundError:
+            pass
+        fdata = fdata + '\n' + data
+        fdata = fdata.split('\n')
+        if len(fdata) >= settings.MAX_LOG_FILE_LINES:
+            fdata.pop(0)
+        fdata = '\n'.join(fdata)
+        with open(settings._outputfile, 'w') as file:
+            file.write(fdata)

src/logger/readline.py

@@ -0,0 +1,37 @@
+'''
+    Onionr - Private P2P Communication
+
+    get a line of input from stdin
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys
+from . import colors, settings
+colors = colors.Colors
+def readline(message = ''):
+    '''
+        Takes in input from the console, not stored in logs
+        message: The message to display before taking input
+    '''
+
+    color = colors.fg.green + colors.bold
+    output = colors.reset + str(color) + '... ' + colors.reset + str(message) + colors.reset
+
+    if not settings.get_settings() & settings.USE_ANSI:
+        output = colors.filter(output)
+
+    sys.stdout.write(output)
+
+    return input()

src/logger/settings.py

@@ -0,0 +1,87 @@
+'''
+    Onionr - Private P2P Communication
+
+    logger settings
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import os
+from utils import identifyhome
+
+data_home = os.environ.get('ONIONR_LOG_DIR', identifyhome.identify_home())
+# Use the bitwise operators to merge these settings
+USE_ANSI = 0b100
+if os.name == 'nt':
+    USE_ANSI = 0b000
+OUTPUT_TO_CONSOLE = 0b010
+OUTPUT_TO_FILE = 0b001
+
+LEVEL_DEBUG = 1
+LEVEL_INFO = 2
+LEVEL_WARN = 3
+LEVEL_ERROR = 4
+LEVEL_FATAL = 5
+LEVEL_IMPORTANT = 6
+
+MAX_LOG_FILE_LINES = 10000
+
+_type = OUTPUT_TO_CONSOLE | USE_ANSI # the default settings for logging
+_level = LEVEL_DEBUG # the lowest level to log
+_outputfile = '%s/onionr.log' % (data_home,) # the file to log to
+
+def set_settings(type):
+    '''
+        Set the settings for the logger using bitwise operators
+    '''
+
+    global _type
+    _type = type
+
+def get_settings():
+    '''
+        Get settings from the logger
+    '''
+
+    return _type
+
+def set_level(level):
+    '''
+        Set the lowest log level to output
+    '''
+
+    global _level
+    _level = level
+
+def get_level()->int:
+    '''
+        Get the lowest log level currently being outputted
+    '''
+
+    return _level
+
+def set_file(outputfile):
+    '''
+        Set the file to output to, if enabled
+    '''
+
+    global _outputfile
+    _outputfile = outputfile
+
+def get_file():
+    '''
+        Get the file to output to
+    '''
+
+    return _outputfile