Merge pull request #12 from beardog108/crypto

Crypto

.gitignore

@@ -9,3 +9,6 @@ onionr/data/*
 onionr/data-backup/*
 onionr/gnupg/*
 run.sh
+onionr/data-encrypted.dat
+onionr/.onionr-lock
+core

Makefile

@@ -16,6 +16,8 @@ uninstall:
 	sudo rm -f /usr/bin/onionr
 
 test:
+	@./RUN-LINUX.sh stop
+	@sleep 1
 	@rm -rf onionr/data-backup
 	@mv onionr/data onionr/data-backup | true > /dev/null 2>&1
 	-@cd onionr; ./tests.py; ./cryptotests.py;
@@ -23,11 +25,16 @@ test:
 	@mv onionr/data-backup onionr/data | true > /dev/null 2>&1
 
 soft-reset:
-	rm -f onionr/data/blocks/*.dat | true > /dev/null 2>&1
+	@echo "Soft-resetting Onionr..."
-	rm -f onionr/data/*.db | true > /dev/null 2>&1
+	rm -f onionr/data/blocks/*.dat onionr/data/*.db | true > /dev/null 2>&1
+	@./RUN-LINUX.sh version | grep -v "Failed" --color=always
 
 reset:
 	@echo "Hard-resetting Onionr..."
 	rm -rf onionr/data/ | true > /dev/null 2>&1
-	@./RUN-LINUX.sh version | grep -v "Failed" --color=always
+	#@./RUN-LINUX.sh version | grep -v "Failed" --color=always
 
+plugins-reset:
+	@echo "Resetting plugins..."
+	rm -rf onionr/data/plugins/ | true > /dev/null 2>&1
+	@./RUN-LINUX.sh version | grep -v "Failed" --color=always

docs/onionr-draft.md

@@ -24,7 +24,13 @@ All traffic is over Tor/I2P, connecting only to Tor onion and I2P hidden service
 
 Onionr nodes use HTTP (over Tor/I2P) to exchange keys, metadata, and blocks. Blocks are identified by their sha3_256 hash. Nodes sync a table of blocks hashes and attempt to download blocks they do not yet have from random peers.
 
-Blocks may be encrypted using Curve25519.
+Blocks may be encrypted using Curve25519 or Salsa20.
+
+Blocks have IDs in the following format:
+
+-Optional hash of public key of publisher (base64)-optional signature (non-optional if publisher is specified) (Base64)-block type-block hash(sha3-256)
+
+pubkeyHash-signature-type-hash
 
 ## Connections
 

onionr/api.py

@@ -20,6 +20,7 @@
 import flask
 from flask import request, Response, abort
 from multiprocessing import Process
+from gevent.wsgi import WSGIServer
 import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config
 
 from core import Core
@@ -32,10 +33,13 @@ class API:
         '''
             Validate that the client token (hmac) matches the given token
         '''
-        if self.clientToken != token:
+        try:
+            if not hmac.compare_digest(self.clientToken, token):
                 return False
             else:
                 return True
+        except TypeError:
+            return False
 
     def __init__(self, debug):
         '''
@@ -63,8 +67,15 @@ class API:
         bindPort = int(config.get('client')['port'])
         self.bindPort = bindPort
         self.clientToken = config.get('client')['client_hmac']
+        self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
+
+        self.mimeType = 'text/plain'
+
+        with open('data/time-bypass.txt', 'w') as bypass:
+            bypass.write(self.timeBypassToken)
+
         if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
-            logger.debug('Your HMAC token: ' + logger.colors.underline + self.clientToken)
+            logger.debug('Your web password (KEEP SECRET): ' + logger.colors.underline + self.clientToken)
 
         if not debug and not self._developmentMode:
             hostNums = [random.randint(1, 255), random.randint(1, 255), random.randint(1, 255)]
@@ -88,41 +99,82 @@ class API:
         def afterReq(resp):
             if not self.requestFailed:
                 resp.headers['Access-Control-Allow-Origin'] = '*'
-            else:
+            #else:
-                resp.headers['server'] = 'Onionr'
+            #    resp.headers['server'] = 'Onionr'
-            resp.headers['Content-Type'] = 'text/plain'
+            resp.headers['Content-Type'] = self.mimeType
-            resp.headers["Content-Security-Policy"] = "default-src 'none'"
+            resp.headers["Content-Security-Policy"] =  "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
             resp.headers['X-Frame-Options'] = 'deny'
             resp.headers['X-Content-Type-Options'] = "nosniff"
+            resp.headers['server'] = 'Onionr'
+
+            # reset to text/plain to help prevent browser attacks
+            if self.mimeType != 'text/plain':
+                self.mimeType = 'text/plain'
 
             return resp
 
         @app.route('/client/')
         def private_handler():
+            if request.args.get('timingToken') is None:
+                timingToken = ''
+            else:
+                timingToken = request.args.get('timingToken')
+            data = request.args.get('data')
+            try:
+                data = data
+            except:
+                data = ''
             startTime = math.floor(time.time())
             # we should keep a hash DB of requests (with hmac) to prevent replays
             action = request.args.get('action')
             #if not self.debug:
             token = request.args.get('token')
+
             if not self.validateToken(token):
                 abort(403)
             self.validateHost('private')
             if action == 'hello':
                 resp = Response('Hello, World! ' + request.host)
             elif action == 'shutdown':
-                request.environ.get('werkzeug.server.shutdown')()
+                # request.environ.get('werkzeug.server.shutdown')()
+                self.http_server.stop()
                 resp = Response('Goodbye')
+            elif action == 'ping':
+                resp = Response('pong')
             elif action == 'stats':
                 resp = Response('me_irl')
+                raise Exception
+            elif action == 'site':
+                block = data
+                siteData = self._core.getData(data)
+                response = 'not found'
+                if siteData != '' and siteData != False:
+                    self.mimeType = 'text/html'
+                    response = siteData.split(b'-', 2)[-1]
+                resp = Response(response)
             else:
                 resp = Response('(O_o) Dude what? (invalid command)')
             endTime = math.floor(time.time())
             elapsed = endTime - startTime
+
+            # if bypass token not used, delay response to prevent timing attacks
+            if not hmac.compare_digest(timingToken, self.timeBypassToken):
                 if elapsed < self._privateDelayTime:
                     time.sleep(self._privateDelayTime - elapsed)
 
             return resp
 
+        @app.route('/')
+        def banner():
+            self.mimeType = 'text/html'
+            self.validateHost('public')
+            try:
+                with open('static-data/index.html', 'r') as html:
+                    resp = Response(html.read())
+            except FileNotFoundError:
+                resp = Response("")
+            return resp
+
         @app.route('/public/')
         def public_handler():
             # Public means it is publicly network accessible
@@ -130,6 +182,10 @@ class API:
             action = request.args.get('action')
             requestingPeer = request.args.get('myID')
             data = request.args.get('data')
+            try:
+                data = data
+            except:
+                data = ''
             if action == 'firstConnect':
                 pass
             elif action == 'ping':
@@ -142,9 +198,25 @@ class API:
                 resp = Response(self._utils.getBlockDBHash())
             elif action == 'getBlockHashes':
                 resp = Response(self._core.getBlockList())
+            elif action == 'directMessage':
+                resp = Response(self._core.handle_direct_connection(data))
+            elif action == 'announce':
+                if data != '':
+                    # TODO: require POW for this
+                    if self._core.addAddress(data):
+                        resp = Response('Success')
+                    else:
+                        resp = Response('')
+                else:
+                    resp = Response('')
             # setData should be something the communicator initiates, not this api
             elif action == 'getData':
-                resp = self._core.getData(data)
+                if self._utils.validateHash(data):
+                    if not os.path.exists('data/blocks/' + data + '.db'):
+                        try:
+                            resp = base64.b64encode(self._core.getData(data))
+                        except TypeError:
+                            resp = ""
                 if resp == False:
                     abort(404)
                     resp = ""
@@ -155,9 +227,8 @@ class API:
                     response = 'none'
                 resp = Response(response)
             elif action == 'kex':
-                response = ','.join(self._core.listPeers())
+                peers = self._core.listPeers(getPow=True)
-                if len(response) == 0:
+                response = ','.join(peers)
-                    response = 'none'
                 resp = Response(response)
             else:
                 resp = Response("")
@@ -185,10 +256,14 @@ class API:
 
             return resp
         if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
-            logger.info('Starting client on ' + self.host + ':' + str(bindPort) + '...')
+            logger.info('Starting client on ' + self.host + ':' + str(bindPort) + '...', timestamp=True)
 
         try:
-            app.run(host=self.host, port=bindPort, debug=True, threaded=True)
+            self.http_server = WSGIServer((self.host, bindPort), app)
+            self.http_server.serve_forever()
+        except KeyboardInterrupt:
+            pass
+            #app.run(host=self.host, port=bindPort, debug=False, threaded=True)
         except Exception as e:
             logger.error(str(e))
             logger.fatal('Failed to start client on ' + self.host + ':' + str(bindPort) + ', exiting...')

onionr/bitpeer

@@ -1 +0,0 @@
-Subproject commit a74e826e9c69e643ead7950f9f76a05ab8664ddc

onionr/btc.py

@@ -1,44 +0,0 @@
-'''
-    Onionr - P2P Microblogging Platform & Social network
-
-    Handle bitcoin operations
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-from bitpeer.node import *
-from bitpeer.storage.shelve import ShelveStorage
-import logging, time
-import socks, sys
-class OnionrBTC:
-	def __init__(self, lastBlock='00000000000000000021ee6242d08e3797764c9258e54e686bc2afff51baf599', lastHeight=510613, torP=9050):
-		stream = logging.StreamHandler()
-		logger = logging.getLogger('halfnode')
-		logger.addHandler(stream)
-		logger.setLevel (10)
-
-		LASTBLOCK = lastBlock
-		LASTBLOCKINDEX = lastHeight
-		self.node = Node ('BTC', ShelveStorage ('data/btc-blocks.db'), lastblockhash=LASTBLOCK, lastblockheight=LASTBLOCKINDEX, torPort=torP)
-
-		self.node.bootstrap ()
-		self.node.connect ()
-		self.node.loop ()
-
-if __name__ == "__main__":
-    torPort = int(sys.argv[1])
-    bitcoin = OnionrBTC(torPort)
-    while True:
-        print(bitcoin.node.getBlockHash(bitcoin.node.getLastBlockHeight())) # Using print on purpose, do not change to logger
-        time.sleep(5)

onionr/communicator.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 '''
-Onionr - P2P Microblogging Platform & Social network.
+    Onionr - P2P Microblogging Platform & Social network.
 
-This file contains both the OnionrCommunicate class for communcating with peers
+    This file contains both the OnionrCommunicate class for communcating with peers
-and code to operate as a daemon, getting commands from the command queue database (see core.Core.daemonQueue)
+    and code to operate as a daemon, getting commands from the command queue database (see core.Core.daemonQueue)
 '''
 '''
     This program is free software: you can redistribute it and/or modify
@@ -19,8 +19,8 @@ and code to operate as a daemon, getting commands from the command queue databas
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import sqlite3, requests, hmac, hashlib, time, sys, os, math, logger, urllib.parse, random
+import sqlite3, requests, hmac, hashlib, time, sys, os, math, logger, urllib.parse, base64, binascii, random, json, threading
-import core, onionrutils, onionrcrypto, onionrproofs, btc, config, onionrplugins as plugins
+import core, onionrutils, onionrcrypto, netcontroller, onionrproofs, config, onionrplugins as plugins
 
 class OnionrCommunicate:
     def __init__(self, debug, developmentMode):
@@ -33,30 +33,37 @@ class OnionrCommunicate:
         self._core = core.Core()
         self._utils = onionrutils.OnionrUtils(self._core)
         self._crypto = onionrcrypto.OnionrCrypto(self._core)
+        self._netController = netcontroller.NetController(0) # arg is the HS port but not needed rn in this file
+
+        self.newHashes = {} # use this to not keep hashes around too long if we cant get their data
+        self.keepNewHash = 12
+        self.ignoredHashes = []
 
         self.highFailureAmount = 7
-        '''
-        logger.info('Starting Bitcoin Node... with Tor socks port:' + str(sys.argv[2]))
-        try:
-            self.bitcoin = btc.OnionrBTC(torP=int(sys.argv[2]))
-        except _gdbm.error:
-            pass
-        logger.info('Bitcoin Node started, on block: ' + self.bitcoin.node.getBlockHash(self.bitcoin.node.getLastBlockHeight()))
-        '''
-        #except:
-        #logger.fatal('Failed to start Bitcoin Node, exiting...')
-        #exit(1)
 
-        blockProcessTimer = 0
+        self.communicatorThreads = 0
-        blockProcessAmount = 5
+        self.maxThreads = 75
-        highFailureTimer = 0
+        self.processBlocksThreads = 0
-        highFailureRate = 10
+        self.lookupBlocksThreads = 0
-        heartBeatTimer = 0
+
-        heartBeatRate = 5
+        self.blocksProcessing = [] # list of blocks currently processing, to avoid trying a block twice at once in 2 seperate threads
-        pexTimer = 5 # How often we should check for new peers
+        self.peerStatus = {} # network actions (active requests) for peers used mainly to prevent conflicting actions in threads
-        pexCount = 0
+
+        self.communicatorTimers = {} # communicator timers, name: rate (in seconds)
+        self.communicatorTimerCounts = {}
+        self.communicatorTimerFuncs = {}
+
+        self.registerTimer('blockProcess', 20)
+        self.registerTimer('highFailure', 10)
+        self.registerTimer('heartBeat', 10)
+        self.registerTimer('pex', 120)
         logger.debug('Communicator debugging enabled.')
-        torID = open('data/hs/hostname').read()
+
+        with open('data/hs/hostname', 'r') as torID:
+            todID = torID.read()
+
+        apiRunningCheckRate = 10
+        apiRunningCheckCount = 0
 
         self.peerData = {} # Session data for peers (recent reachability, speed, etc)
 
@@ -69,37 +76,337 @@ class OnionrCommunicate:
         while True:
             command = self._core.daemonQueue()
             # Process blocks based on a timer
-            blockProcessTimer += 1
+            self.timerTick() 
-            heartBeatTimer += 1
+            # TODO: migrate below if statements to be own functions which are called in the above timerTick() function
-            pexCount += 1
+            if self.communicatorTimers['highFailure'] == self.communicatorTimerCounts['highFailure']:
-            if highFailureTimer == highFailureRate:
+                self.communicatorTimerCounts['highFailure'] = 0
-                highFailureTimer = 0
                 for i in self.peerData:
-                    if self.peerData[i]['failCount'] == self.highFailureAmount:
+                    if self.peerData[i]['failCount'] >= self.highFailureAmount:
                         self.peerData[i]['failCount'] -= 1
-            if pexTimer == pexCount:
+            if self.communicatorTimers['pex'] == self.communicatorTimerCounts['pex']:
-                self.getNewPeers()
+                pT1 = threading.Thread(target=self.getNewPeers, name="pT1")
-                pexCount = 0
+                pT1.start()
-            if heartBeatRate == heartBeatTimer:
+                pT2 = threading.Thread(target=self.getNewPeers, name="pT2")
+                pT2.start()
+                self.communicatorTimerCounts['pex'] = 0# TODO: do not reset timer if low peer count
+            if self.communicatorTimers['heartBeat'] == self.communicatorTimerCounts['heartBeat']:
                 logger.debug('Communicator heartbeat')
-                heartBeatTimer = 0
+                self.communicatorTimerCounts['heartBeat'] = 0
-            if blockProcessTimer == blockProcessAmount:
+            if self.communicatorTimers['blockProcess'] == self.communicatorTimerCounts['blockProcess']:
-                self.lookupBlocks()
+                lT1 = threading.Thread(target=self.lookupBlocks, name="lt1", args=(True,))
-                self.processBlocks()
+                lT2 = threading.Thread(target=self.lookupBlocks, name="lt2", args=(True,))
-                blockProcessTimer = 0
+                lT3 = threading.Thread(target=self.lookupBlocks, name="lt3", args=(True,))
+                lT4 = threading.Thread(target=self.lookupBlocks, name="lt4", args=(True,))
+                pbT1 = threading.Thread(target=self.processBlocks, name='pbT1', args=(True,))
+                pbT2 = threading.Thread(target=self.processBlocks, name='pbT2', args=(True,))
+                pbT3 = threading.Thread(target=self.processBlocks, name='pbT3', args=(True,))
+                pbT4 = threading.Thread(target=self.processBlocks, name='pbT4', args=(True,))
+                if (self.maxThreads - 8) >= threading.active_count():
+                    lT1.start()
+                    lT2.start()
+                    lT3.start()
+                    lT4.start()
+                    pbT1.start()
+                    pbT2.start()
+                    pbT3.start()
+                    pbT4.start()
+                    self.communicatorTimerCounts['blockProcess'] = 0
+                else:
+                    logger.debug(threading.active_count())
+                    logger.debug('Too many threads.')
             if command != False:
                 if command[0] == 'shutdown':
-                    logger.info('Daemon recieved exit command.')
+                    logger.info('Daemon received exit command.', timestamp=True)
                     break
+                elif command[0] == 'announceNode':
+                    announceAttempts = 3
+                    announceAttemptCount = 0
+                    announceVal = False
+                    logger.info('Announcing node to ' + command[1], timestamp=True)
+                    while not announceVal:
+                        announceAttemptCount += 1
+                        announceVal = self.performGet('announce', command[1], data=self._core.hsAdder.replace('\n', ''), skipHighFailureAddress=True)
+                        logger.info(announceVal)
+                        if announceAttemptCount >= announceAttempts:
+                            logger.warn('Unable to announce to ' + command[1])
+                            break
+                elif command[0] == 'runCheck':
+                    logger.info('Status check; looks good.')
+                    open('data/.runcheck', 'w+').close()
+                elif command[0] == 'kex':
+                    self.pexCount = pexTimer - 1
+                elif command[0] == 'event':
+                    # todo
+                    pass
+                elif command[0] == 'checkCallbacks':
+                    try:
+                        data = json.loads(command[1])
+
+                        logger.info('Checking for callbacks with connection %s...' % data['id'])
+
+                        self.check_callbacks(data, config.get('dc_execcallbacks', True))
+
+                        events.event('incoming_direct_connection', data = {'callback' : True, 'communicator' : self, 'data' : data})
+                    except Exception as e:
+                        logger.error('Failed to interpret callbacks for checking', e)
+                elif command[0] == 'incomingDirectConnection':
+                    try:
+                        data = json.loads(command[1])
+
+                        logger.info('Handling incoming connection %s...' % data['id'])
+
+                        self.incoming_direct_connection(data)
+
+                        events.event('incoming_direct_connection', data = {'callback' : False, 'communicator' : self, 'data' : data})
+                    except Exception as e:
+                        logger.error('Failed to handle callbacks for checking', e)
+
+            apiRunningCheckCount += 1
+
+            # check if local API is up
+            if apiRunningCheckCount > apiRunningCheckRate:
+                if self._core._utils.localCommand('ping') != 'pong':
+                    for i in range(4):
+                        if self._utils.localCommand('ping') == 'pong':
+                            apiRunningCheckCount = 0
+                            break # break for loop
                         time.sleep(1)
+                    else:
+                        # This executes if the api is NOT detected to be running
+                        logger.error('Daemon detected API crash (or otherwise unable to reach API after long time), stopping...')
+                        break # break main daemon loop
+                apiRunningCheckCount = 0
+
+            time.sleep(1)
+
+        self._netController.killTor()
+        return
+
+    future_callbacks = {}
+    connection_handlers = {}
+    id_peer_cache = {}
+
+    def registerTimer(self, timerName, rate, timerFunc=None):
+        '''Register a communicator timer'''
+        self.communicatorTimers[timerName] = rate
+        self.communicatorTimerCounts[timerName] = 0
+        self.communicatorTimerFuncs[timerName] = timerFunc
+    
+    def timerTick(self):
+        '''Increments timers "ticks" and calls funcs if applicable'''
+        tName = ''
+        for i in self.communicatorTimers.items():
+            tName = i[0]
+            self.communicatorTimerCounts[tName] += 1
+
+            if self.communicatorTimerCounts[tName] == self.communicatorTimers[tName]:
+                try:
+                    self.communicatorTimerFuncs[tName]()
+                except TypeError:
+                    pass
+                else:
+                    self.communicatorTimerCounts[tName] = 0
+
+
+    def get_connection_handlers(self, name = None):
+        '''
+            Returns a list of callback handlers by name, or, if name is None, it returns all handlers.
+        '''
+
+        if name is None:
+            return self.connection_handlers
+        elif name in self.connection_handlers:
+            return self.connection_handlers[name]
+        else:
+            return list()
+
+    def add_connection_handler(self, name, handler):
+        '''
+            Adds a function to be called when an connection that is NOT a callback is received.
+            Takes in the name of the communication type and the handler as input
+        '''
+
+        if not name in self.connection_handlers:
+            self.connection_handlers[name] = list()
+
+        self.connection_handlers[name].append(handler)
+
+        return
+
+    def remove_connection_handler(self, name, handler = None):
+        '''
+            Removes a connection handler if specified, or removes all by name
+        '''
+
+        if handler is None:
+            if name in self.connection_handlers:
+                self.connection_handlers[name].remove(handler)
+        elif name in self.connection_handlers:
+            del self.connection_handlers[name]
+
+        return
+
+
+    def set_callback(self, identifier, callback):
+        '''
+            (Over)writes a callback by communication identifier
+        '''
+
+        if not callback is None:
+            self.future_callbacks[identifier] = callback
+            return True
+
+        return False
+
+    def unset_callback(self, identifier):
+        '''
+            Unsets a callback by communication identifier, if set
+        '''
+
+        if identifier in future_callbacks:
+            del self.future_callbacks[identifier]
+            return True
+
+        return False
+
+    def get_callback(self, identifier):
+        '''
+            Returns a callback by communication identifier if set, or None
+        '''
+
+        if identifier in self.future_callbacks:
+            return self.future_callbacks[id]
+
+        return None
+
+    def direct_connect(self, peer, data = None, callback = None, log = True):
+        '''
+            Communicates something directly with the client
+
+            - `peer` should obviously be the peer id to request.
+            - `data` should be a dict (NOT str), with the parameter "type"
+              ex. {'type': 'sendMessage', 'content': 'hey, this is a dm'}
+              In that dict, the key 'token' must NEVER be set. If it is, it will
+              be overwritten.
+            - if `callback` is set to a function, it will call that function
+              back if/when the client the request is sent to decides to respond.
+              Do NOT depend on a response, because users can configure their
+              clients not to respond to this type of request.
+            - `log` is set to True by default-- what this does is log the
+              request for debug purposes. Should be False for sensitive actions.
+        '''
+
+        # TODO: Timing attack prevention
+        try:
+            # does not need to be secure random, only used for keeping track of async responses
+            # Actually, on second thought, it does need to be secure random. Otherwise, if it is predictable, someone could trigger arbitrary callbacks that have been saved on the local node, wrecking all kinds of havoc. Better just to keep it secure random.
+            identifier = self._utils.token(32)
+            if 'id' in data:
+                identifier = data['id']
+
+            if not identifier in id_peer_cache:
+                id_peer_cache[identifier] = peer
+
+            if type(data) == str:
+                # if someone inputs a string instead of a dict, it will assume it's the type
+                data = {'type' : data}
+
+            data['id'] = identifier
+            data['token'] = '' # later put PoW stuff here or whatever is needed
+            data_str = json.dumps(data)
+
+            events.event('outgoing_direct_connection', data = {'callback' : True, 'communicator' : self, 'data' : data, 'id' : identifier, 'token' : token, 'peer' : peer, 'callback' : callback, 'log' : log})
+
+            logger.debug('Direct connection (identifier: "%s"): %s' % (identifier, data_str))
+            try:
+                self.performGet('directMessage', peer, data_str)
+            except:
+                logger.warn('Failed to connect to peer: "%s".' % str(peer))
+                return False
+
+            if not callback is None:
+                self.set_callback(identifier, callback)
+
+            return True
+        except Exception as e:
+            logger.warn('Unknown error, failed to execute direct connect (peer: "%s").' % str(peer), e)
+
+        return False
+
+    def direct_connect_response(self, identifier, data, peer = None, callback = None, log = True):
+        '''
+            Responds to a previous connection. Hostname will be pulled from id_peer_cache if not specified in `peer` parameter.
+
+            If yet another callback is requested, it can be put in the `callback` parameter.
+        '''
+
+        if config.get('dc_response', True):
+            data['id'] = identifier
+            data['sender'] = open('data/hs/hostname').read()
+            data['callback'] = True
+
+            if (origin is None) and (identifier in id_peer_cache):
+                origin = id_peer_cache[identifier]
+
+            if not identifier in id_peer_cache:
+                id_peer_cache[identifier] = peer
+
+            if origin is None:
+                logger.warn('Failed to identify peer for connection %s' % str(identifier))
+                return False
+            else:
+                return self.direct_connect(peer, data = data, callback = callback, log = log)
+        else:
+            logger.warn('Node tried to respond to direct connection id %s, but it was rejected due to `dc_response` restriction.' % str(identifier))
+            return False
+
+
+    def check_callbacks(self, data, execute = True, remove = True):
+        '''
+            Check if a callback is set, and if so, execute it
+        '''
+
+        try:
+            if type(data) is str:
+                data = json.loads(data)
+
+            if 'id' in data: # TODO: prevent enumeration, require extra PoW
+                identifier = data['id']
+
+                if identifier in self.future_callbacks:
+                    if execute:
+                        self.get_callback(identifier)(data)
+                        logger.debug('Request callback "%s" executed.' % str(identifier))
+                    if remove:
+                        self.unset_callback(identifier)
+
+                    return True
+
+                logger.warn('Unable to find request callback for ID "%s".' % str(identifier))
+            else:
+                logger.warn('Unable to identify callback request, `id` parameter missing: %s' % json.dumps(data))
+        except Exception as e:
+            logger.warn('Unknown error, failed to execute direct connection callback (peer: "%s").' % str(peer), e)
+
+        return False
+
+    def incoming_direct_connection(self, data):
+        '''
+            This code is run whenever there is a new incoming connection.
+        '''
+
+        if 'type' in data and data['type'] in self.connection_handlers:
+            for handler in self.get_connection_handlers(name):
+                handler(data)
 
         return
 
     def getNewPeers(self):
         '''
-            Get new peers
+            Get new peers and ed25519 keys
         '''
-        peersCheck = 5 # Amount of peers to ask for new peers + keys
+
+        peersCheck = 1 # Amount of peers to ask for new peers + keys
         peersChecked = 0
         peerList = list(self._core.listAdders()) # random ordered list of peers
         newKeys = []
@@ -114,119 +421,258 @@ class OnionrCommunicate:
             peersCheck = len(peerList)
 
         while peersCheck > peersChecked:
+            #i = secrets.randbelow(maxN) # cant use prior to 3.6
             i = random.randint(0, maxN)
-            logger.info('Using ' + peerList[i] + ' to find new peers')
+
+            try:
+                if self.peerStatusTaken(peerList[i], 'pex') or self.peerStatusTaken(peerList[i], 'kex'):
+                    continue
+            except IndexError:
+                pass
+
+            logger.info('Using %s to find new peers...' % peerList[i], timestamp=True)
+
             try:
                 newAdders = self.performGet('pex', peerList[i], skipHighFailureAddress=True)
+                if not newAdders is False: # keep the is False thing in there, it might not be bool
+                    logger.debug('Attempting to merge address: %s' % str(newAdders))
                     self._utils.mergeAdders(newAdders)
             except requests.exceptions.ConnectionError:
-                logger.info(peerList[i] + ' connection failed')
+                logger.info('%s connection failed' % peerList[i], timestamp=True)
                 continue
             else:
                 try:
-                    logger.info('Using ' + peerList[i] + ' to find new keys')
+                    logger.info('Using %s to find new keys...' % peerList[i])
                     newKeys = self.performGet('kex', peerList[i], skipHighFailureAddress=True)
+                    logger.debug('Attempting to merge pubkey: %s' % str(newKeys))
                     # TODO: Require keys to come with POW token (very large amount of POW)
                     self._utils.mergeKeys(newKeys)
                 except requests.exceptions.ConnectionError:
-                    logger.info(peerList[i] + ' connection failed')
+                    logger.info('%s connection failed' % peerList[i], timestamp=True)
                     continue
                 else:
                     peersChecked += 1
         return
 
-    def lookupBlocks(self):
+    def lookupBlocks(self, isThread=False):
         '''
             Lookup blocks and merge new ones
         '''
+        if isThread:
+            self.lookupBlocksThreads += 1
         peerList = self._core.listAdders()
-        blocks = ''
+        blockList = list()
+
         for i in peerList:
+            if self.peerStatusTaken(i, 'getBlockHashes') or self.peerStatusTaken(i, 'getDBHash'):
+                continue
+            try:
+                if self.peerData[i]['failCount'] >= self.highFailureAmount:
+                    continue
+            except KeyError:
+                pass
+
             lastDB = self._core.getAddressInfo(i, 'DBHash')
+
             if lastDB == None:
-                logger.debug('Fetching hash from ' + i + ' No previous known.')
+                logger.debug('Fetching hash from %s, no previous known.' % str(i))
             else:
-                logger.debug('Fetching hash from ' + str(i) + ', ' + lastDB + ' last known')
+                logger.debug('Fetching hash from %s, %s last known' % (str(i), str(lastDB)))
+
             currentDB = self.performGet('getDBHash', i)
+
             if currentDB != False:
-                logger.debug(i + " hash db (from request): " + currentDB)
+                logger.debug('%s hash db (from request): %s' % (str(i), str(currentDB)))
             else:
-                logger.warn("Error getting hash db status for " + i)
+                logger.warn('Failed to get hash db status for %s' % str(i))
+
             if currentDB != False:
                 if lastDB != currentDB:
-                    logger.debug('Fetching hash from ' + i + ' - ' + currentDB + ' current hash.')
+                    logger.debug('Fetching hash from %s - %s current hash.' % (str(i), currentDB))
-                    blocks += self.performGet('getBlockHashes', i)
+                    try:
+                        blockList.append(self.performGet('getBlockHashes', i))
+                    except TypeError:
+                        logger.warn('Failed to get data hash from %s' % str(i))
+                        self.peerData[i]['failCount'] -= 1
                 if self._utils.validateHash(currentDB):
                     self._core.setAddressInfo(i, "DBHash", currentDB)
-        if len(blocks.strip()) != 0:
+
-            logger.debug('BLOCKS:' + blocks)
+        if len(blockList) != 0:
-        blockList = blocks.split('\n')
+            pass
+
         for i in blockList:
             if len(i.strip()) == 0:
                 continue
+            try:
                 if self._utils.hasBlock(i):
                     continue
-            logger.debug('Exchanged block (blockList): ' + i)
+            except:
+                logger.warn('Invalid hash') # TODO: move below validate hash check below
+                pass
+            if i in self.ignoredHashes:
+                continue
+
+            #logger.debug('Exchanged block (blockList): ' + i)
             if not self._utils.validateHash(i):
                 # skip hash if it isn't valid
-                logger.warn('Hash ' + i + ' is not valid')
+                logger.warn('Hash %s is not valid' % str(i))
                 continue
             else:
-                logger.debug('Adding ' +  i + ' to hash database...')
+                self.newHashes[i] = 0
+                logger.debug('Adding %s to hash database...' % str(i))
                 self._core.addToBlockDB(i)
-
+        self.lookupBlocksThreads -= 1
         return
 
-    def processBlocks(self):
+    def processBlocks(self, isThread=False):
         '''
             Work with the block database and download any missing blocks
 
             This is meant to be called from the communicator daemon on its timer.
         '''
-
+        if isThread:
-        for i in self._core.getBlockList(True).split("\n"):
+            self.processBlocksThreads += 1
+        for i in self._core.getBlockList(unsaved = True):
             if i != "":
-                logger.warn('UNSAVED BLOCK: ' + i)
+                if i in self.blocksProcessing or i in self.ignoredHashes:
+                    #logger.debug('already processing ' + i)
+                    continue
+                else:
+                    self.blocksProcessing.append(i)
+                try:
+                    self.newHashes[i]
+                except KeyError:
+                    self.newHashes[i] = 0
+
+                # check if a new hash has been around too long, delete it from database and add it to ignore list
+                if self.newHashes[i] >= self.keepNewHash:
+                    logger.warn('Ignoring block %s because it took to long to get valid data.' % str(i))
+                    del self.newHashes[i]
+                    self._core.removeBlock(i)
+                    self.ignoredHashes.append(i)
+                    continue
+
+                self.newHashes[i] += 1
+                logger.warn('Block is unsaved: %s' % str(i))
                 data = self.downloadBlock(i)
 
+                # if block was successfully gotten (hash already verified)
+                if data:
+                    del self.newHashes[i] # remove from probation list
+
+                    # deal with block metadata
+                    blockContent = self._core.getData(i)
+                    try:
+                        blockContent = blockContent.encode()
+                    except AttributeError:
+                        pass
+                    try:
+                        #blockMetadata = json.loads(self._core.getData(i)).split('}')[0] + '}'
+                        blockMetadata = json.loads(blockContent[:blockContent.find(b'\n')].decode())
+                        try:
+                            blockMeta2 = json.loads(blockMetadata['meta'])
+                        except KeyError:
+                            blockMeta2 = {'type': ''}
+                            pass
+                        blockContent = blockContent[blockContent.find(b'\n') + 1:]
+                        try:
+                            blockContent = blockContent.decode()
+                        except AttributeError:
+                            pass
+
+                        if not self._crypto.verifyPow(blockContent, blockMeta2):
+                            logger.warn("%s has invalid or insufficient proof of work token, deleting..." % str(i))
+                            self._core.removeBlock(i)
+                            continue
+                        else:
+                            if (('sig' in blockMetadata) and ('id' in blockMeta2)): # id doesn't exist in blockMeta2, so this won't workin the first place
+
+                                #blockData = json.dumps(blockMetadata['meta']) + blockMetadata[blockMetadata.rfind(b'}') + 1:]
+
+                                creator = self._utils.getPeerByHashId(blockMeta2['id'])
+                                try:
+                                    creator = creator.decode()
+                                except AttributeError:
+                                    pass
+
+                                if self._core._crypto.edVerify(blockMetadata['meta'] + blockContent, creator, blockMetadata['sig'], encodedData=True):
+                                    logger.info('%s was signed' % str(i))
+                                    self._core.updateBlockInfo(i, 'sig', 'true')
+                                else:
+                                    logger.warn('%s has an invalid signature' % str(i))
+                                    self._core.updateBlockInfo(i, 'sig', 'false')
+                        try:
+                            logger.info('Block type is %s' % str(blockMeta2['type']))
+                            self._core.updateBlockInfo(i, 'dataType', blockMeta2['type'])
+                            self.removeBlockFromProcessingList(i)
+                            self.removeBlockFromProcessingList(i)
+                        except KeyError:
+                            logger.warn('Block has no type')
+                            pass
+                    except json.decoder.JSONDecodeError:
+                        logger.warn('Could not decode block metadata')
+                        self.removeBlockFromProcessingList(i)
+        self.processBlocksThreads -= 1
         return
 
-    def downloadBlock(self, hash):
+    def removeBlockFromProcessingList(self, block):
+        return block in blocksProcessing
+
+    def downloadBlock(self, hash, peerTries=3):
         '''
             Download a block from random order of peers
         '''
 
+        retVal = False
         peerList = self._core.listAdders()
         blocks = ''
+        peerTryCount = 0
+
         for i in peerList:
-            hasher = hashlib.sha3_256()
+            try:
-            data = self.performGet('getData', i, hash)
+                if self.peerData[i]['failCount'] >= self.highFailureAmount:
-            if data == False or len(data) > 10000000:
                     continue
-            hasher.update(data.encode())
+            except KeyError:
+                pass
+            if peerTryCount >= peerTries:
+                break
+
+            hasher = hashlib.sha3_256()
+            data = self.performGet('getData', i, hash, skipHighFailureAddress=True)
+
+            if data == False or len(data) > 10000000 or data == '':
+                peerTryCount += 1
+                continue
+
+            try:
+                data = base64.b64decode(data)
+            except binascii.Error:
+                data = b''
+
+            hasher.update(data)
             digest = hasher.hexdigest()
+
             if type(digest) is bytes:
                 digest = digest.decode()
+
             if digest == hash.strip():
                 self._core.setData(data)
-                if data.startswith('-txt-'):
+                logger.info('Successfully obtained data for %s' % str(hash), timestamp=True)
-                    self._core.setBlockType(hash, 'txt')
+                retVal = True
-                logger.info('Successfully obtained data for ' + hash)
+                break
-                if len(data) < 120:
-                    logger.debug('Block text:\n' + data)
             else:
-                logger.warn("Failed to validate " + hash)
+                logger.warn("Failed to validate %s -- hash calculated was %s" % (hash, digest))
+                peerTryCount += 1
 
-        return
+        return retVal
 
     def urlencode(self, data):
         '''
             URL encodes the data
         '''
-
         return urllib.parse.quote_plus(data)
 
-    def performGet(self, action, peer, data=None, skipHighFailureAddress=False, peerType='tor'):
+    def performGet(self, action, peer, data=None, skipHighFailureAddress=False, peerType='tor', selfCheck=True):
         '''
             Performs a request to a peer through Tor or i2p (currently only Tor)
         '''
@@ -234,9 +680,16 @@ class OnionrCommunicate:
         if not peer.endswith('.onion') and not peer.endswith('.onion/'):
             raise PeerError('Currently only Tor .onion peers are supported. You must manually specify .onion')
 
+        if len(self._core.hsAdder.strip()) == 0:
+            raise Exception("Could not perform self address check in performGet due to not knowing our address")
+        if selfCheck:
+            if peer.replace('/', '') == self._core.hsAdder:
+                logger.warn('Tried to performGet to own hidden service, but selfCheck was not set to false')
+                return
+
         # Store peer in peerData dictionary (non permanent)
         if not peer in self.peerData:
-            self.peerData[peer] = {'connectCount': 0, 'failCount': 0, 'lastConnectTime': math.floor(time.time())}
+            self.peerData[peer] = {'connectCount': 0, 'failCount': 0, 'lastConnectTime': self._utils.getEpoch()}
         socksPort = sys.argv[2]
         '''We use socks5h to use tor as DNS'''
         proxies = {'http': 'socks5://127.0.0.1:' + str(socksPort), 'https': 'socks5://127.0.0.1:' + str(socksPort)}
@@ -247,13 +700,14 @@ class OnionrCommunicate:
         try:
             if skipHighFailureAddress and self.peerData[peer]['failCount'] > self.highFailureAmount:
                 retData = False
-                logger.debug('Skipping ' + peer + ' because of high failure rate')
+                logger.debug('Skipping %s because of high failure rate.' % peer)
             else:
-                logger.debug('Contacting ' + peer + ' on port ' + socksPort)
+                self.peerStatus[peer] = action
+                logger.debug('Contacting %s on port %s' % (peer, str(socksPort)))
                 r = requests.get(url, headers=headers, proxies=proxies, timeout=(15, 30))
                 retData = r.text
         except requests.exceptions.RequestException as e:
-            logger.warn(action + " failed with peer " + peer + ": " + str(e))
+            logger.debug("%s failed with peer %s" % (action, peer))
             retData = False
 
         if not retData:
@@ -261,9 +715,20 @@ class OnionrCommunicate:
         else:
             self.peerData[peer]['connectCount'] += 1
             self.peerData[peer]['failCount'] -= 1
-            self.peerData[peer]['lastConnectTime'] = math.floor(time.time())
+            self.peerData[peer]['lastConnectTime'] = self._utils.getEpoch()
+            self._core.setAddressInfo(peer, 'lastConnect', self._utils.getEpoch())
         return retData
 
+    def peerStatusTaken(self, peer, status):
+        '''
+            Returns if we are currently performing a specific action with a peer.
+        '''
+        try:
+            if self.peerStatus[peer] == status:
+                return True
+        except KeyError:
+            pass
+        return False
 
 shouldRun = False
 debug = True

onionr/core.py

@@ -17,12 +17,11 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import sqlite3, os, sys, time, math, base64, tarfile, getpass, simplecrypt, hashlib, nacl, logger
+import sqlite3, os, sys, time, math, base64, tarfile, getpass, simplecrypt, hashlib, nacl, logger, json, netcontroller, math
 #from Crypto.Cipher import AES
 #from Crypto import Random
-import netcontroller
 
-import onionrutils, onionrcrypto, btc
+import onionrutils, onionrcrypto, onionrproofs, onionrevents as events
 
 if sys.version_info < (3, 6):
     try:
@@ -36,11 +35,16 @@ class Core:
         '''
             Initialize Core Onionr library
         '''
+        try:
             self.queueDB = 'data/queue.db'
             self.peerDB = 'data/peers.db'
             self.blockDB = 'data/blocks.db'
             self.blockDataLocation = 'data/blocks/'
             self.addressDB = 'data/address.db'
+            self.hsAdder = ''
+
+            self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
+            self.bootstrapList = []
 
             if not os.path.exists('data/'):
                 os.mkdir('data/')
@@ -49,44 +53,96 @@ class Core:
             if not os.path.exists(self.blockDB):
                 self.createBlockDB()
 
+            if os.path.exists('data/hs/hostname'):
+                with open('data/hs/hostname', 'r') as hs:
+                    self.hsAdder = hs.read()
+
+            # Load bootstrap address list
+            if os.path.exists(self.bootstrapFileLocation):
+                with open(self.bootstrapFileLocation, 'r') as bootstrap:
+                    bootstrap = bootstrap.read()
+                for i in bootstrap.split('\n'):
+                    self.bootstrapList.append(i)
+            else:
+                logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation)
+
             self._utils = onionrutils.OnionrUtils(self)
             # Initialize the crypto object
             self._crypto = onionrcrypto.OnionrCrypto(self)
 
+        except Exception as error:
+            logger.error('Failed to initialize core Onionr library.', error=error)
+            logger.fatal('Cannot recover from error.')
+            sys.exit(1)
         return
 
-    def addPeer(self, peerID, name=''):
+    def addPeer(self, peerID, powID, name=''):
         '''
             Adds a public key to the key database (misleading function name)
-
-            DOES NO SAFETY CHECKS if the ID is valid, but prepares the insertion
         '''
         # This function simply adds a peer to the DB
         if not self._utils.validatePubKey(peerID):
             return False
+        if sys.getsizeof(powID) > 60:
+            logger.warn("POW token for pubkey base64 representation exceeded 60 bytes")
+            return False
+
         conn = sqlite3.connect(self.peerDB)
+        hashID = self._crypto.pubKeyHashID(peerID)
         c = conn.cursor()
-        t = (peerID, name, 'unknown')
+        t = (peerID, name, 'unknown', hashID, powID)
-        c.execute('INSERT INTO peers (id, name, dateSeen) VALUES(?, ?, ?);', t)
+
+        for i in c.execute("SELECT * FROM PEERS where id = '" + peerID + "';"):
+            try:
+                if i[0] == peerID:
+                    conn.close()
+                    return False
+            except ValueError:
+                pass
+            except IndexError:
+                pass
+        c.execute('INSERT INTO peers (id, name, dateSeen, pow, hashID) VALUES(?, ?, ?, ?, ?);', t)
         conn.commit()
         conn.close()
+
         return True
 
     def addAddress(self, address):
-        '''Add an address to the address database (only tor currently)'''
+        '''
+            Add an address to the address database (only tor currently)
+        '''
         if self._utils.validateID(address):
             conn = sqlite3.connect(self.addressDB)
             c = conn.cursor()
+            # check if address is in database
+            # this is safe to do because the address is validated above, but we strip some chars here too just in case
+            address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '')
+            for i in c.execute("SELECT * FROM adders where address = '" + address + "';"):
+                try:
+                    if i[0] == address:
+                        logger.warn('Not adding existing address')
+                        conn.close()
+                        return False
+                except ValueError:
+                    pass
+                except IndexError:
+                    pass
+
             t = (address, 1)
             c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t)
             conn.commit()
             conn.close()
+
+            events.event('address_add', data = {'address': address}, onionr = None)
+
             return True
         else:
             return False
 
     def removeAddress(self, address):
-        '''Remove an address from the address database'''
+        '''
+            Remove an address from the address database
+        '''
         if self._utils.validateID(address):
             conn = sqlite3.connect(self.addressDB)
             c = conn.cursor()
@@ -94,10 +150,29 @@ class Core:
             c.execute('Delete from adders where address=?;', t)
             conn.commit()
             conn.close()
+
+            events.event('address_remove', data = {'address': address}, onionr = None)
+
             return True
         else:
             return False
 
+    def removeBlock(self, block):
+        '''
+            remove a block from this node
+        '''
+        if self._utils.validateHash(block):
+            conn = sqlite3.connect(self.blockDB)
+            c = conn.cursor()
+            t = (block,)
+            c.execute('Delete from hashes where hash=?;', t)
+            conn.commit()
+            conn.close()
+            try:
+                os.remove('data/blocks/' + block + '.dat')
+            except FileNotFoundError:
+                pass
+
     def createAddressDB(self):
         '''
             Generate the address database
@@ -116,7 +191,8 @@ class Core:
             speed int,
             success int,
             DBHash text,
-            failure int
+            failure int,
+            lastConnect int
             );
         ''')
         conn.commit()
@@ -137,7 +213,10 @@ class Core:
             forwardKey text,
             dateSeen not null,
             bytesStored int,
-            trust int);
+            trust int,
+            pubkeyExchanged int,
+            hashID text,
+            pow text not null);
         ''')
         conn.commit()
         conn.close()
@@ -153,6 +232,8 @@ class Core:
             dataType     - data type of the block
             dataFound    - if the data has been found for the block
             dataSaved    - if the data has been saved for the block
+            sig    - optional signature by the author (not optional if author is specified)
+            author       - multi-round partial sha3-256 hash of authors public key
         '''
         if os.path.exists(self.blockDB):
             raise Exception("Block database already exists")
@@ -164,14 +245,17 @@ class Core:
             decrypted int,
             dataType text,
             dataFound int,
-            dataSaved int);
+            dataSaved int,
+            sig text,
+            author text
+            );
         ''')
         conn.commit()
         conn.close()
 
         return
 
-    def addToBlockDB(self, newHash, selfInsert=False):
+    def addToBlockDB(self, newHash, selfInsert=False, dataSaved=False):
         '''
             Add a hash value to the block db
 
@@ -183,24 +267,25 @@ class Core:
             return
         conn = sqlite3.connect(self.blockDB)
         c = conn.cursor()
-        currentTime = math.floor(time.time())
+        currentTime = self._utils.getEpoch()
-        if selfInsert:
+        if selfInsert or dataSaved:
             selfInsert = 1
         else:
             selfInsert = 0
-        data = (newHash, currentTime, 0, '', 0, selfInsert)
+        data = (newHash, currentTime, '', selfInsert)
-        c.execute('INSERT INTO hashes VALUES(?, ?, ?, ?, ?, ?);', data)
+        c.execute('INSERT INTO hashes (hash, dateReceived, dataType, dataSaved) VALUES(?, ?, ?, ?);', data)
         conn.commit()
         conn.close()
 
         return
 
-    def getData(self,hash):
+    def getData(self, hash):
         '''
             Simply return the data associated to a hash
         '''
         try:
-            dataFile = open(self.blockDataLocation + hash + '.dat')
+            # logger.debug('Opening %s' % (str(self.blockDataLocation) + str(hash) + '.dat'))
+            dataFile = open(self.blockDataLocation + hash + '.dat', 'rb')
             data = dataFile.read()
             dataFile.close()
         except FileNotFoundError:
@@ -212,8 +297,10 @@ class Core:
         '''
             Set the data assciated with a hash
         '''
-        data = data.encode()
+        data = data
         hasher = hashlib.sha3_256()
+        if not type(data) is bytes:
+            data = data.encode()
         hasher.update(data)
         dataHash = hasher.hexdigest()
         if type(dataHash) is bytes:
@@ -223,8 +310,8 @@ class Core:
             pass # TODO: properly check if block is already saved elsewhere
             #raise Exception("Data is already set for " + dataHash)
         else:
-            blockFile = open(blockFileName, 'w')
+            blockFile = open(blockFileName, 'wb')
-            blockFile.write(data.decode())
+            blockFile.write(data)
             blockFile.close()
 
         conn = sqlite3.connect(self.blockDB)
@@ -296,6 +383,8 @@ class Core:
         conn.commit()
         conn.close()
 
+        events.event('queue_pop', data = {'data': retData}, onionr = None)
+
         return retData
 
     def daemonQueueAdd(self, command, data=''):
@@ -303,7 +392,7 @@ class Core:
             Add a command to the daemon queue, used by the communication daemon (communicator.py)
         '''
         # Intended to be used by the web server
-        date = math.floor(time.time())
+        date = self._utils.getEpoch()
         conn = sqlite3.connect(self.queueDB)
         c = conn.cursor()
         t = (command, data, date)
@@ -311,6 +400,8 @@ class Core:
         conn.commit()
         conn.close()
 
+        events.event('queue_push', data = {'command': command, 'data': data}, onionr = None)
+
         return
 
     def clearDaemonQueue(self):
@@ -320,11 +411,12 @@ class Core:
         conn = sqlite3.connect(self.queueDB)
         c = conn.cursor()
         try:
-            c.execute('delete from commands;')
+            c.execute('DELETE FROM commands;')
             conn.commit()
         except:
             pass
         conn.close()
+        events.event('queue_clear', onionr = None)
 
         return
 
@@ -344,7 +436,7 @@ class Core:
         conn.close()
         return addressList
 
-    def listPeers(self, randomOrder=True):
+    def listPeers(self, randomOrder=True, getPow=False):
         '''
             Return a list of public keys (misleading function name)
 
@@ -352,15 +444,29 @@ class Core:
         '''
         conn = sqlite3.connect(self.peerDB)
         c = conn.cursor()
+        payload = ""
         if randomOrder:
-            peers = c.execute('SELECT * FROM peers ORDER BY RANDOM();')
+            payload = 'SELECT * FROM peers ORDER BY RANDOM();'
         else:
-            peers = c.execute('SELECT * FROM peers;')
+            payload = 'SELECT * FROM peers;'
         peerList = []
-        for i in peers:
+        for i in c.execute(payload):
-            peerList.append(i[2])
+            try:
+                if len(i[0]) != 0:
+                    if getPow:
+                        peerList.append(i[0] + '-' + i[1])
+                    else:
+                        peerList.append(i[0])
+            except TypeError:
+                pass
+        if getPow:
+            try:
+                peerList.append(self._crypto.pubKey + '-' + self._crypto.pubKeyPowToken)
+            except TypeError:
+                pass
+        else:
+            peerList.append(self._crypto.pubKey)
         conn.close()
-
         return peerList
 
     def getPeerInfo(self, peer, info):
@@ -369,17 +475,18 @@ class Core:
 
             id text             0
             name text,          1
-            pubkey text,        2
+            adders text,        2
-            adders text,        3
+            forwardKey text,    3
-            forwardKey text,    4
+            dateSeen not null,  4
-            dateSeen not null,  5
+            bytesStored int,    5
-            bytesStored int,    6
+            trust int           6
-            trust int           7
+            pubkeyExchanged int 7
+            hashID text         8
         '''
         conn = sqlite3.connect(self.peerDB)
         c = conn.cursor()
         command = (peer,)
-        infoNumbers = {'id': 0, 'name': 1, 'pubkey': 2, 'adders': 3, 'forwardKey': 4, 'dateSeen': 5, 'bytesStored': 6, 'trust': 7}
+        infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'forwardKey': 3, 'dateSeen': 4, 'bytesStored': 5, 'trust': 6, 'pubkeyExchanged': 7, 'hashID': 8}
         info = infoNumbers[info]
         iterCount = 0
         retVal = ''
@@ -420,11 +527,12 @@ class Core:
             success int, 4
             DBHash text, 5
             failure int 6
+            lastConnect 7
         '''
         conn = sqlite3.connect(self.addressDB)
         c = conn.cursor()
         command = (address,)
-        infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'DBHash': 5, 'failure': 6}
+        infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'DBHash': 5, 'failure': 6, 'lastConnect': 7}
         info = infoNumbers[info]
         iterCount = 0
         retVal = ''
@@ -446,29 +554,62 @@ class Core:
         c = conn.cursor()
         command = (data, address)
         # TODO: validate key on whitelist
-        if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'DBHash', 'failure'):
+        if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'DBHash', 'failure', 'lastConnect'):
             raise Exception("Got invalid database key when setting address info")
         c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command)
         conn.commit()
         conn.close()
         return
 
-    def getBlockList(self, unsaved=False):
+    def handle_direct_connection(self, data):
+        '''
+            Handles direct messages
+        '''
+        try:
+            data = json.loads(data)
+
+            # TODO: Determine the sender, verify, etc
+            if ('callback' in data) and (data['callback'] is True):
+                # then this is a response to the message we sent earlier
+                self.daemonQueueAdd('checkCallbacks', json.dumps(data))
+            else:
+                # then we should handle it and respond accordingly
+                self.daemonQueueAdd('incomingDirectConnection', json.dumps(data))
+        except Exception as e:
+            logger.warn('Failed to handle incoming direct message: %s' % str(e))
+
+        return
+
+    def getBlockList(self, unsaved = False): # TODO: Use unsaved
         '''
             Get list of our blocks
         '''
         conn = sqlite3.connect(self.blockDB)
         c = conn.cursor()
-        retData = ''
         if unsaved:
-            execute = 'SELECT hash FROM hashes WHERE dataSaved != 1;'
+            execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
         else:
-            execute = 'SELECT hash FROM hashes;'
+            execute = 'SELECT hash FROM hashes ORDER BY RANDOM();'
+        rows = list()
         for row in c.execute(execute):
             for i in row:
-                retData += i + "\n"
+                rows.append(i)
 
-        return retData
+        return rows
+
+    def getBlockDate(self, blockHash):
+        '''
+            Returns the date a block was received
+        '''
+        conn = sqlite3.connect(self.blockDB)
+        c = conn.cursor()
+        execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
+        args = (blockHash,)
+        for row in c.execute(execute, args):
+            for i in row:
+                return int(i)
+
+        return None
 
     def getBlocksByType(self, blockType):
         '''
@@ -476,14 +617,14 @@ class Core:
         '''
         conn = sqlite3.connect(self.blockDB)
         c = conn.cursor()
-        retData = ''
         execute = 'SELECT hash FROM hashes WHERE dataType=?;'
         args = (blockType,)
+        rows = list()
         for row in c.execute(execute, args):
             for i in row:
-                retData += i + "\n"
+                rows.append(i)
 
-        return retData.split('\n')
+        return rows
 
     def setBlockType(self, hash, blockType):
         '''
@@ -495,5 +636,111 @@ class Core:
         c.execute("UPDATE hashes SET dataType='" + blockType + "' WHERE hash = '" + hash + "';")
         conn.commit()
         conn.close()
+        return
+
+    def updateBlockInfo(self, hash, key, data):
+        '''
+            sets info associated with a block
+        '''
+
+        if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author'):
+            return False
+
+        conn = sqlite3.connect(self.blockDB)
+        c = conn.cursor()
+        args = (data, hash)
+        c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args)
+        conn.commit()
+        conn.close()
+        return True
+
+    def insertBlock(self, data, header='txt', sign=False):
+        '''
+            Inserts a block into the network
+        '''
+
+        powProof = onionrproofs.POW(data)
+        powToken = ''
+        # wait for proof to complete
+        try:
+            while True:
+                powToken = powProof.getResult()
+                if powToken == False:
+                    time.sleep(0.3)
+                    continue
+                powHash = powToken[0]
+                powToken = base64.b64encode(powToken[1])
+                try:
+                    powToken = powToken.decode()
+                except AttributeError:
+                    pass
+                finally:
+                    break
+        except KeyboardInterrupt:
+            logger.warn("Got keyboard interrupt while working on inserting block, stopping.")
+            powProof.shutdown()
+            return ''
+
+        try:
+            data.decode()
+        except AttributeError:
+            data = data.encode()
+
+        retData = ''
+        metadata = {'type': header, 'powHash': powHash, 'powToken': powToken}
+        sig = {}
+
+        metadata = json.dumps(metadata)
+        metadata = metadata.encode()
+        signature = ''
+
+        if sign:
+            signature = self._crypto.edSign(metadata + b'\n' + data, self._crypto.privKey, encodeResult=True)
+            ourID = self._crypto.pubKeyHashID()
+            # Convert from bytes on some py versions?
+            try:
+                ourID = ourID.decode()
+            except AttributeError:
+                pass
+        metadata = {'sig': signature, 'meta': metadata.decode()}
+        metadata = json.dumps(metadata)
+        metadata = metadata.encode()
+
+        if len(data) == 0:
+            logger.error('Will not insert empty block')
+        else:
+            addedHash = self.setData(metadata + b'\n' + data)
+            self.addToBlockDB(addedHash, selfInsert=True)
+            self.setBlockType(addedHash, header)
+            retData = addedHash
+        return retData
+
+    def introduceNode(self):
+        '''
+            Introduces our node into the network by telling X many nodes our HS address
+        '''
+
+        if(self._utils.isCommunicatorRunning()):
+            announceAmount = 2
+            nodeList = self.listAdders()
+
+            if len(nodeList) == 0:
+                for i in self.bootstrapList:
+                    if self._utils.validateID(i):
+                        self.addAddress(i)
+                        nodeList.append(i)
+
+            if announceAmount > len(nodeList):
+                announceAmount = len(nodeList)
+
+            for i in range(announceAmount):
+                self.daemonQueueAdd('announceNode', nodeList[i])
+
+            events.event('introduction', onionr = None)
+
+            return True
+        else:
+            logger.error('Onionr daemon is not running.')
+            return False
 
         return

onionr/cryptotests.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+'''
+    Onionr - P2P Microblogging Platform & Social network
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import unittest, sys, os, core, onionrcrypto, logger
+
+class OnionrCryptoTests(unittest.TestCase):
+    def testSymmetric(self):
+        dataString = "this is a secret message"
+        dataBytes = dataString.encode()
+        myCore = core.Core()
+        crypto = onionrcrypto.OnionrCrypto(myCore)
+        key = key = b"tttttttttttttttttttttttttttttttt"
+
+        logger.info("Encrypting: " + dataString, timestamp=True)
+        encrypted = crypto.symmetricEncrypt(dataString, key, returnEncoded=True)
+        logger.info(encrypted, timestamp=True)
+        logger.info('Decrypting encrypted string:', timestamp=True)
+        decrypted = crypto.symmetricDecrypt(encrypted, key, encodedMessage=True)
+        logger.info(decrypted, timestamp=True)
+        self.assertTrue(True)
+if __name__ == "__main__":
+    unittest.main()

onionr/gui.py

@@ -1,70 +0,0 @@
-#!/usr/bin/python
-'''
-    Onionr - P2P Microblogging Platform & Social network
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-from tkinter import *
-import os, sqlite3, core
-class OnionrGUI:
-    def __init__(self, myCore):
-        self.root = Tk()
-        self.myCore = myCore # onionr core
-        self.root.title("PyOnionr")
-
-        w = Label(self.root, text="Onionr", width=10)
-        w.config(font=("Sans-Serif", 22))
-        w.pack()
-        scrollbar = Scrollbar(self.root)
-        scrollbar.pack(side=RIGHT, fill=Y)
-
-        self.listedBlocks = []
-
-        idText = open('./data/hs/hostname', 'r').read()
-        idLabel = Label(self.root, text="ID: " + idText)
-        idLabel.pack(pady=5)
-
-        self.sendEntry = Entry(self.root)
-        sendBtn = Button(self.root, text='Send Message', command=self.sendMessage)
-        self.sendEntry.pack()
-        sendBtn.pack()
-
-        self.listbox = Listbox(self.root, yscrollcommand=scrollbar.set, height=15)
-
-        #listbox.insert(END, str(i))
-        self.listbox.pack(fill=BOTH)
-
-        scrollbar.config(command=self.listbox.yview)
-        self.root.after(2000, self.update)
-        self.root.mainloop() 
-
-    def sendMessage(self):
-        messageToAdd = '-txt-' + self.sendEntry.get()
-        addedHash = self.myCore.setData(messageToAdd)
-        self.myCore.addToBlockDB(addedHash, selfInsert=True)
-        self.myCore.setBlockType(addedHash, 'txt')
-        self.sendEntry.delete(0, END)
-
-    def update(self):
-        for i in self.myCore.getBlocksByType('txt'):
-            if i.strip() == '' or i in self.listedBlocks:
-                continue
-            blockFile = open('./data/blocks/' + i + '.dat')
-            self.listbox.insert(END, str(blockFile.read().replace('-txt-', '')))
-            blockFile.close()
-            self.listedBlocks.append(i)
-            self.listbox.see(END)
-        blocksList = os.listdir('./data/blocks/') # dir is your directory path
-        number_blocks = len(blocksList)
-
-        self.root.after(10000, self.update)

onionr/logger.py

@@ -18,7 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 
-import re, sys
+import re, sys, time, traceback
 
 class colors:
     '''
@@ -134,15 +134,18 @@ def raw(data):
         with open(_outputfile, "a+") as f:
             f.write(colors.filter(data) + '\n')
 
-def log(prefix, data, color = ''):
+def log(prefix, data, color = '', timestamp=True):
     '''
         Logs the data
         prefix : The prefix to the output
         data   : The actual data to output
         color  : The color to output before the data
     '''
+    curTime = ''
+    if timestamp:
+        curTime = time.strftime("%m-%d %H:%M:%S") + ' '
 
-    output = colors.reset + str(color) + '[' + colors.bold + str(prefix) + colors.reset + str(color) + '] ' + str(data) + colors.reset
+    output = colors.reset + str(color) + '[' + colors.bold + str(prefix) + colors.reset + str(color) + '] ' + curTime + str(data) + colors.reset
     if not get_settings() & USE_ANSI:
         output = colors.filter(output)
 
@@ -198,26 +201,38 @@ def confirm(default = 'y', message = 'Are you sure %s? '):
         return default == 'y'
 
 # debug: when there is info that could be useful for debugging purposes only
-def debug(data):
+def debug(data, timestamp=True):
     if get_level() <= LEVEL_DEBUG:
-        log('/', data)
+        log('/', data, timestamp=timestamp)
 
 # info: when there is something to notify the user of, such as the success of a process
-def info(data):
+def info(data, timestamp=False):
     if get_level() <= LEVEL_INFO:
-        log('+', data, colors.fg.green)
+        log('+', data, colors.fg.green, timestamp=timestamp)
 
 # warn: when there is a potential for something bad to happen
-def warn(data):
+def warn(data, timestamp=True):
     if get_level() <= LEVEL_WARN:
-        log('!', data, colors.fg.orange)
+        log('!', data, colors.fg.orange, timestamp=timestamp)
 
 # error: when only one function, module, or process of the program encountered a problem and must stop
-def error(data):
+def error(data, error=None, timestamp=True):
     if get_level() <= LEVEL_ERROR:
-        log('-', data, colors.fg.red)
+        log('-', data, colors.fg.red, timestamp=timestamp)
+    if not error is None:
+        debug('Error: ' + str(error) + parse_error())
 
-# fatal: when the something so bad has happened that the prorgam must stop
+# fatal: when the something so bad has happened that the program must stop
-def fatal(data):
+def fatal(data, timestamp=True):
     if get_level() <= LEVEL_FATAL:
-        log('#', data, colors.bg.red + colors.fg.green + colors.bold)
+        log('#', data, colors.bg.red + colors.fg.green + colors.bold, timestamp=timestamp)
+
+# returns a formatted error message
+def parse_error():
+    details = traceback.extract_tb(sys.exc_info()[2])
+    output = ''
+
+    for line in details:
+        output += '\n    ... module %s in  %s:%i' % (line[2], line[0], line[1])
+
+    return output

onionr/netcontroller.py

@@ -52,6 +52,7 @@ class NetController:
         torrcData = '''SocksPort ''' + str(self.socksPort) + '''
 HiddenServiceDir data/hs/
 HiddenServicePort 80 127.0.0.1:''' + str(self.hsPort) + '''
+DataDirectory data/tordata/
         '''
         torrc = open(self.torConfigLocation, 'w')
         torrc.write(torrcData)
@@ -88,6 +89,7 @@ HiddenServicePort 80 127.0.0.1:''' + str(self.hsPort) + '''
             torVersion.kill()
 
         # wait for tor to get to 100% bootstrap
+        try:
             for line in iter(tor.stdout.readline, b''):
                 if 'Bootstrapped 100%: Done' in line.decode():
                     break
@@ -96,8 +98,11 @@ HiddenServicePort 80 127.0.0.1:''' + str(self.hsPort) + '''
             else:
                 logger.fatal('Failed to start Tor. Try killing any other Tor processes owned by this user.')
                 return False
+        except KeyboardInterrupt:
+            logger.fatal("Got keyboard interrupt")
+            return False
 
-        logger.info('Finished starting Tor')
+        logger.info('Finished starting Tor', timestamp=True)
         self.readyState = True
 
         myID = open('data/hs/hostname', 'r')

onionr/onionr.py

@@ -20,8 +20,15 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import sys, os, base64, random, getpass, shutil, subprocess, requests, time, platform
+
-import api, core, gui, config, logger, onionrplugins as plugins
+import sys
+if sys.version_info[0] == 2 or sys.version_info[1] < 5:
+    print('Error, Onionr requires Python 3.4+')
+    sys.exit(1)
+import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass
+from threading import Thread
+import api, core, config, logger, onionrplugins as plugins, onionrevents as events
+import onionrutils
 from onionrutils import OnionrUtils
 from netcontroller import NetController
 
@@ -30,14 +37,11 @@ try:
 except ImportError:
     raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)")
 
-ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - onionr.voidnet.tech'
+ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech'
 ONIONR_VERSION = '0.0.0' # for debugging and stuff
-API_VERSION = '1' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes knows how to communicate without learning too much information about you.
+API_VERSION = '2' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes knows how to communicate without learning too much information about you.
 
 class Onionr:
-    cmds = {}
-    cmdhelp = {}
-
     def __init__(self):
         '''
             Main Onionr class. This is for the CLI program, and does not handle much of the logic.
@@ -51,8 +55,18 @@ class Onionr:
 
         # Load global configuration data
 
-        exists = os.path.exists(config.get_config_file())
+        data_exists = os.path.exists('data/')
-        config.set_config({'devmode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}}) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
+
+        if not data_exists:
+            os.mkdir('data/')
+
+        if os.path.exists('static-data/default_config.json'):
+            config.set_config(json.loads(open('static-data/default_config.json').read())) # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
+        else:
+            # the default config file doesn't exist, try hardcoded config
+            config.set_config({'devmode': True, 'log': {'file': {'output': True, 'path': 'data/output.log'}, 'console': {'output': True, 'color': True}}})
+        if not data_exists:
+            config.save()
         config.reload() # this will read the configuration file into memory
 
         settings = 0b000
@@ -65,7 +79,7 @@ class Onionr:
             logger.set_file(config.get('log', {'file': {'path': 'data/output.log'}})['file']['path'])
         logger.set_settings(settings)
 
-        if config.get('devmode', True):
+        if str(config.get('devmode', True)).lower() == 'true':
             self._developmentMode = True
             logger.set_level(logger.LEVEL_DEBUG)
         else:
@@ -87,12 +101,31 @@ class Onionr:
                 if os.path.exists('data/'):
                     break
                 else:
-                    logger.error('Failed to decrypt: ' + result[1])
+                    logger.error('Failed to decrypt: ' + result[1], timestamp = False)
         else:
-            if not os.path.exists('data/'):
+            # If data folder does not exist
-                os.mkdir('data/')
+            if not data_exists:
+                if not os.path.exists('data/blocks/'):
                     os.mkdir('data/blocks/')
 
+            # Copy default plugins into plugins folder
+            if not os.path.exists(plugins.get_plugins_folder()):
+                if os.path.exists('static-data/default-plugins/'):
+                    names = [f for f in os.listdir("static-data/default-plugins/") if not os.path.isfile(f)]
+                    shutil.copytree('static-data/default-plugins/', plugins.get_plugins_folder())
+
+                    # Enable plugins
+                    for name in names:
+                        if not name in plugins.get_enabled_plugins():
+                            plugins.enable(name, self)
+
+        for name in plugins.get_enabled_plugins():
+            if not os.path.exists(plugins.get_plugin_data_folder(name)):
+                try:
+                    os.mkdir(plugins.get_plugin_data_folder(name))
+                except:
+                    plugins.disable(name, onionr = self, stop_event = False)
+
         if not os.path.exists(self.onionrCore.peerDB):
             self.onionrCore.createPeerDB()
             pass
@@ -101,7 +134,7 @@ class Onionr:
 
         # Get configuration
 
-        if not exists:
+        if not data_exists:
             # Generate default config
             # Hostname should only be set if different from 127.x.x.x. Important for DNS rebinding attack prevention.
             if self.debug:
@@ -111,7 +144,7 @@ class Onionr:
                     randomPort = random.randint(1024, 65535)
                     if self.onionrUtils.checkPort(randomPort):
                         break
-            config.set('client', {'participate': 'true', 'client_hmac': base64.b64encode(os.urandom(32)).decode('utf-8'), 'port': randomPort, 'api_version': API_VERSION}, True)
+            config.set('client', {'participate': 'true', 'client_hmac': base64.b16encode(os.urandom(32)).decode('utf-8'), 'port': randomPort, 'api_version': API_VERSION}, True)
 
         self.cmds = {
             '': self.showHelpSuggestion,
@@ -120,6 +153,8 @@ class Onionr:
             'config': self.configure,
             'start': self.start,
             'stop': self.killDaemon,
+            'status': self.showStats,
+            'statistics': self.showStats,
             'stats': self.showStats,
 
             'enable-plugin': self.enablePlugin,
@@ -134,9 +169,12 @@ class Onionr:
             'reloadplugin': self.reloadPlugin,
             'reload-plugins': self.reloadPlugin,
             'reloadplugins': self.reloadPlugin,
+            'create-plugin': self.createPlugin,
+            'createplugin': self.createPlugin,
+            'plugin-create': self.createPlugin,
 
-            'listpeers': self.listPeers,
+            'listkeys': self.listKeys,
-            'list-peers': self.listPeers,
+            'list-keys': self.listKeys,
 
             'addmsg': self.addMessage,
             'addmessage': self.addMessage,
@@ -144,13 +182,20 @@ class Onionr:
             'add-message': self.addMessage,
             'pm': self.sendEncrypt,
 
-            'gui': self.openGUI,
+            'getpms': self.getPMs,
+            'get-pms': self.getPMs,
 
             'addpeer': self.addPeer,
             'add-peer': self.addPeer,
             'add-address': self.addAddress,
+            'add-addr': self.addAddress,
+            'addaddr': self.addAddress,
             'addaddress': self.addAddress,
+            'addfile': self.addFile,
 
+            'importblocks': self.onionrUtils.importNewBlocks,
+
+            'introduce': self.onionrCore.introduceNode,
             'connect': self.addAddress
         }
 
@@ -164,13 +209,20 @@ class Onionr:
             'enable-plugin': 'Enables and starts a plugin',
             'disable-plugin': 'Disables and stops a plugin',
             'reload-plugin': 'Reloads a plugin',
+            'create-plugin': 'Creates directory structure for a plugin',
+            'add-peer': 'Adds a peer to database',
             'list-peers': 'Displays a list of peers',
-            'add-peer': 'Adds a peer (?)',
             'add-msg': 'Broadcasts a message to the Onionr network',
             'pm': 'Adds a private message to block',
-            'gui': 'Opens a graphical interface for Onionr'
+            'get-pms': 'Shows private messages sent to you',
+            'addfile': 'Create an Onionr block from a file',
+            'importblocks': 'import blocks from the disk (Onionr is transport-agnostic!)',
+            'introduce': 'Introduce your node to the public Onionr network',
         }
 
+        # initialize plugins
+        events.event('init', onionr = self, threaded = False)
+
         command = ''
         try:
             command = sys.argv[1].lower()
@@ -197,10 +249,16 @@ class Onionr:
         return self.cmdhelp
 
     def addCommand(self, command, function):
-        cmds[str(command).lower()] = function
+        self.cmds[str(command).lower()] = function
 
     def addHelp(self, command, description):
-        cmdhelp[str(command).lower()] = str(description)
+        self.cmdhelp[str(command).lower()] = str(description)
+
+    def delCommand(self, command):
+        return self.cmds.pop(str(command).lower(), None)
+
+    def delHelp(self, command):
+        return self.cmdhelp.pop(str(command).lower(), None)
 
     def configure(self):
         '''
@@ -223,6 +281,7 @@ class Onionr:
         '''
             Executes a command
         '''
+
         argument = argument[argument.startswith('--') and len('--'):] # remove -- if it starts with it
 
         # define commands
@@ -231,6 +290,8 @@ class Onionr:
         command = commands.get(argument, self.notFound)
         command()
 
+        return
+
     '''
         THIS SECTION DEFINES THE COMMANDS
     '''
@@ -239,25 +300,29 @@ class Onionr:
         '''
             Displays the Onionr version
         '''
-        logger.info('Onionr ' + ONIONR_VERSION + ' (' + platform.machine() + ') - API v' + API_VERSION)
+
+        logger.info('Onionr %s (%s) - API v%s' % (ONIONR_VERSION, platform.machine(), API_VERSION))
         if verbosity >= 1:
             logger.info(ONIONR_TAGLINE)
         if verbosity >= 2:
-            logger.info('Running on ' + platform.platform() + ' ' + platform.release())
+            logger.info('Running on %s %s' % (platform.platform(), platform.release()))
+
+        return
 
     def sendEncrypt(self):
         '''
             Create a private message and send it
         '''
 
-        while True:
+        invalidID = True
+        while invalidID:
             try:
                 peer = logger.readline('Peer to send to: ')
             except KeyboardInterrupt:
                 break
             else:
-                if self.onionrUtils.validateID(peer):
+                if self.onionrUtils.validatePubKey(peer):
-                    break
+                    invalidID = False
                 else:
                     logger.error('Invalid peer ID')
         else:
@@ -266,23 +331,16 @@ class Onionr:
             except KeyboardInterrupt:
                 pass
             else:
-                logger.info("Sending message to " + peer)
+                logger.info("Sending message to: " + logger.colors.underline + peer)
                 self.onionrUtils.sendPM(peer, message)
 
 
-    def openGUI(self):
+    def listKeys(self):
         '''
-            Opens a graphical interface for Onionr
+            Displays a list of keys (used to be called peers) (?)
         '''
 
-        gui.OnionrGUI(self.onionrCore)
+        logger.info('Public keys in database:\n')
-
-    def listPeers(self):
-        '''
-            Displays a list of peers (?)
-        '''
-
-        logger.info('Peer list:\n')
         for i in self.onionrCore.listPeers():
             logger.info(i)
 
@@ -302,7 +360,10 @@ class Onionr:
         return
 
     def addAddress(self):
-        '''Adds a Onionr node address'''
+        '''
+            Adds a Onionr node address
+        '''
+
         try:
             newAddress = sys.argv[2]
         except:
@@ -310,28 +371,40 @@ class Onionr:
         else:
             logger.info("Adding address: " + logger.colors.underline + newAddress)
             if self.onionrCore.addAddress(newAddress):
-                logger.info("Successfully added address")
+                logger.info("Successfully added address.")
             else:
-                logger.warn("Unable to add address")
+                logger.warn("Unable to add address.")
 
         return
 
-    def addMessage(self):
+    def addMessage(self, header="txt"):
         '''
             Broadcasts a message to the Onionr network
         '''
 
         while True:
-            messageToAdd = '-txt-' + logger.readline('Broadcast message to network: ')
+            try:
+                messageToAdd = logger.readline('Broadcast message to network: ')
                 if len(messageToAdd) >= 1:
                     break
-
+            except KeyboardInterrupt:
-        addedHash = self.onionrCore.setData(messageToAdd)
-        self.onionrCore.addToBlockDB(addedHash, selfInsert=True)
-        self.onionrCore.setBlockType(addedHash, 'txt')
-
                 return
 
+        #addedHash = self.onionrCore.setData(messageToAdd)
+        addedHash = self.onionrCore.insertBlock(messageToAdd, header='txt')
+        #self.onionrCore.addToBlockDB(addedHash, selfInsert=True)
+        #self.onionrCore.setBlockType(addedHash, 'txt')
+        if addedHash != '':
+            logger.info("Message inserted as as block %s" % addedHash)
+        return
+
+    def getPMs(self):
+        '''
+            display PMs sent to us
+        '''
+
+        self.onionrUtils.loadPMs()
+
     def enablePlugin(self):
         '''
             Enables and starts the given plugin
@@ -339,10 +412,10 @@ class Onionr:
 
         if len(sys.argv) >= 3:
             plugin_name = sys.argv[2]
-            logger.info('Enabling plugin \"' + plugin_name + '\"...')
+            logger.info('Enabling plugin "%s"...' % plugin_name)
-            plugins.enable(plugin_name)
+            plugins.enable(plugin_name, self)
         else:
-            logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin>')
+            logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]))
 
         return
 
@@ -353,10 +426,10 @@ class Onionr:
 
         if len(sys.argv) >= 3:
             plugin_name = sys.argv[2]
-            logger.info('Disabling plugin \"' + plugin_name + '\"...')
+            logger.info('Disabling plugin "%s"...' % plugin_name)
-            plugins.disable(plugin_name)
+            plugins.disable(plugin_name, self)
         else:
-            logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin>')
+            logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]))
 
         return
 
@@ -367,12 +440,43 @@ class Onionr:
 
         if len(sys.argv) >= 3:
             plugin_name = sys.argv[2]
-            logger.info('Reloading plugin \"' + plugin_name + '\"...')
+            logger.info('Reloading plugin "%s"...' % plugin_name)
-            plugins.stop(plugin_name)
+            plugins.stop(plugin_name, self)
-            plugins.start(plugin_name)
+            plugins.start(plugin_name, self)
         else:
             logger.info('Reloading all plugins...')
-            plugins.reload()
+            plugins.reload(self)
+
+        return
+
+    def createPlugin(self):
+        '''
+            Creates the directory structure for a plugin name
+        '''
+
+        if len(sys.argv) >= 3:
+            try:
+                plugin_name = re.sub('[^0-9a-zA-Z]+', '', str(sys.argv[2]).lower())
+
+                if not plugins.exists(plugin_name):
+                    logger.info('Creating plugin "%s"...' % plugin_name)
+
+                    os.makedirs(plugins.get_plugins_folder(plugin_name))
+                    with open(plugins.get_plugins_folder(plugin_name) + '/main.py', 'a') as main:
+                        main.write(open('static-data/default_plugin.py').read().replace('$user', os.getlogin()).replace('$date', datetime.datetime.now().strftime('%Y-%m-%d')).replace('$name', plugin_name))
+
+                    with open(plugins.get_plugins_folder(plugin_name) + '/info.json', 'a') as main:
+                        main.write(json.dumps({'author' : 'anonymous', 'description' : 'the default description of the plugin', 'version' : '1.0'}))
+
+                    logger.info('Enabling plugin "%s"...' % plugin_name)
+                    plugins.enable(plugin_name, self)
+                else:
+                    logger.warn('Cannot create plugin directory structure; plugin "%s" exists.' % plugin_name)
+
+            except Exception as e:
+                logger.error('Failed to create plugin directory structure.', e)
+        else:
+            logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]))
 
         return
 
@@ -381,7 +485,7 @@ class Onionr:
             Displays a "command not found" message
         '''
 
-        logger.error('Command not found.')
+        logger.error('Command not found.', timestamp = False)
 
     def showHelpSuggestion(self):
         '''
@@ -390,7 +494,7 @@ class Onionr:
 
         logger.info('Do ' + logger.colors.bold + sys.argv[0] + ' --help' + logger.colors.reset + logger.colors.fg.green + ' for Onionr help.')
 
-    def start(self):
+    def start(self, input = False):
         '''
             Starts the Onionr daemon
         '''
@@ -402,7 +506,9 @@ class Onionr:
                 lockFile = open('.onionr-lock', 'w')
                 lockFile.write('')
                 lockFile.close()
+            self.running = True
             self.daemon()
+            self.running = False
             if not self.debug and not self._developmentMode:
                 os.remove('.onionr-lock')
 
@@ -423,6 +529,7 @@ class Onionr:
             time.sleep(1)
             subprocess.Popen(["./communicator.py", "run", str(net.socksPort)])
             logger.debug('Started communicator')
+            events.event('daemon_start', onionr = self)
         api.API(self.debug)
 
         return
@@ -432,7 +539,9 @@ class Onionr:
             Shutdown the Onionr daemon
         '''
 
-        logger.warn('Killing the running daemon')
+        logger.warn('Killing the running daemon...', timestamp = False)
+        try:
+            events.event('daemon_stop', onionr = self)
             net = NetController(config.get('client')['port'])
             try:
                 self.onionrUtils.localCommand('shutdown')
@@ -440,6 +549,8 @@ class Onionr:
                 pass
             self.onionrCore.daemonQueueAdd('shutdown')
             net.killTor()
+        except Exception as e:
+            logger.error('Failed to shutdown daemon.', error = e, timestamp = False)
 
         return
 
@@ -448,6 +559,54 @@ class Onionr:
             Displays statistics and exits
         '''
 
+        try:
+            # define stats messages here
+            messages = {
+                # info about local client
+                'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 2) else logger.colors.fg.red + 'Offline'),
+                'Public Key' : self.onionrCore._crypto.pubKey,
+                'Address' : self.get_hostname(),
+
+                # file and folder size stats
+                'div1' : True, # this creates a solid line across the screen, a div
+                'Total Block Size' : onionrutils.humanSize(onionrutils.size('data/blocks/')),
+                'Total Plugin Size' : onionrutils.humanSize(onionrutils.size('data/plugins/')),
+                'Log File Size' : onionrutils.humanSize(onionrutils.size('data/output.log')),
+
+                # count stats
+                'div2' : True,
+                'Known Peers Count' : str(len(self.onionrCore.listPeers())),
+                'Enabled Plugins Count' : str(len(config.get('plugins')['enabled'])) + ' / ' + str(len(os.listdir('data/plugins/')))
+            }
+
+            # color configuration
+            colors = {
+                'title' : logger.colors.bold,
+                'key' : logger.colors.fg.lightgreen,
+                'val' : logger.colors.fg.green,
+                'border' : logger.colors.fg.lightblue,
+
+                'reset' : logger.colors.reset
+            }
+
+            # pre-processing
+            maxlength = 0
+            for key, val in messages.items():
+                if not (type(val) is bool and val is True):
+                    maxlength = max(len(key), maxlength)
+
+            # generate stats table
+            logger.info(colors['title'] + 'Onionr v%s Statistics' % ONIONR_VERSION + colors['reset'])
+            logger.info(colors['border'] + '─' * (maxlength + 1) + '┐' + colors['reset'])
+            for key, val in messages.items():
+                if not (type(val) is bool and val is True):
+                    logger.info(colors['key'] + str(key).rjust(maxlength) + colors['reset'] + colors['border'] + ' │ ' + colors['reset'] + colors['val'] + str(val) + colors['reset'])
+                else:
+                    logger.info(colors['border'] + '─' * (maxlength + 1) + '┤' + colors['reset'])
+            logger.info(colors['border'] + '─' * (maxlength + 1) + '┘' + colors['reset'])
+        except Exception as e:
+            logger.error('Failed to generate statistics table.', error = e, timestamp = False)
+
         return
 
     def showHelp(self, command = None):
@@ -462,13 +621,35 @@ class Onionr:
                 self.showHelp(cmd)
         elif not command is None:
             if command.lower() in helpmenu:
-                logger.info(logger.colors.bold + command  + logger.colors.reset + logger.colors.fg.blue + ' : ' + logger.colors.reset +  helpmenu[command.lower()])
+                logger.info(logger.colors.bold + command  + logger.colors.reset + logger.colors.fg.blue + ' : ' + logger.colors.reset +  helpmenu[command.lower()], timestamp = False)
             else:
-                logger.warn(logger.colors.bold + command  + logger.colors.reset + logger.colors.fg.blue + ' : ' + logger.colors.reset + 'No help menu entry was found')
+                logger.warn(logger.colors.bold + command  + logger.colors.reset + logger.colors.fg.blue + ' : ' + logger.colors.reset + 'No help menu entry was found', timestamp = False)
         else:
             self.version(0)
             for command, helpmessage in helpmenu.items():
                 self.showHelp(command)
         return
 
+    def get_hostname(self):
+        try:
+            with open('./data/hs/hostname', 'r') as hostname:
+                return hostname.read().strip()
+        except Exception:
+            return None
+
+    def addFile(self):
+        '''command to add a file to the onionr network'''
+        if len(sys.argv) >= 2:
+            newFile = sys.argv[2]
+            logger.info('Attempting to add file...')
+            try:
+                with open(newFile, 'rb') as new:
+                    new = new.read()
+            except FileNotFoundError:
+                logger.warn('That file does not exist. Improper path?')
+            else:
+                logger.debug(new)
+                logger.info(self.onionrCore.insertBlock(new, header='bin'))
+
+
 Onionr()

onionr/onionrblockapi.py

@@ -0,0 +1,443 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network.
+
+    This class contains the OnionrBlocks class which is a class for working with Onionr blocks
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import core as onionrcore, logger
+import json, os, datetime
+
+class Block:
+    def __init__(self, hash = None, core = None):
+        '''
+            Initializes Onionr
+
+            Inputs:
+            - hash (str): the hash of the block to be imported, if any
+            - core (Core/str):
+              - if (Core): this is the Core instance to be used, don't create a new one
+              - if (str): treat `core` as the block content, and instead, treat `hash` as the block type
+
+            Outputs:
+            - (Block): the new Block instance
+        '''
+
+        # input from arguments
+        if (type(hash) == str) and (type(core) == str):
+            self.btype = hash
+            self.bcontent = core
+            self.hash = None
+            self.core = None
+        else:
+            self.btype = ''
+            self.bcontent = ''
+            self.hash = hash
+            self.core = core
+
+        # initialize variables
+        self.valid = True
+        self.raw = None
+        self.powHash = None
+        self.powToken = None
+        self.signed = False
+        self.signature = None
+        self.signedData = None
+        self.blockFile = None
+        self.bheader = {}
+        self.bmetadata = {}
+
+        # handle arguments
+        if self.getCore() is None:
+            self.core = onionrcore.Core()
+        if not self.getHash() is None:
+            self.update()
+
+    # logic
+
+    def update(self, data = None, file = None):
+        '''
+            Loads data from a block in to the current object.
+
+            Inputs:
+            - data (str):
+              - if None: will load from file by hash
+              - else: will load from `data` string
+            - file (str):
+              - if None: will load from file specified in this parameter
+              - else: will load from wherever block is stored by hash
+
+            Outputs:
+            - (bool): indicates whether or not the operation was successful
+        '''
+
+        try:
+            # import from string
+            blockdata = data
+
+            # import from file
+            if blockdata is None:
+                filelocation = file
+
+                if filelocation is None:
+                    if self.getHash() is None:
+                        return False
+
+                    filelocation = 'data/blocks/%s.dat' % self.getHash()
+
+                blockdata = open(filelocation, 'rb').read().decode('utf-8')
+
+                self.blockFile = filelocation
+            else:
+                self.blockFile = None
+
+            # parse block
+            self.raw = str(blockdata)
+            self.bheader = json.loads(self.getRaw()[:self.getRaw().index('\n')])
+            self.bcontent = self.getRaw()[self.getRaw().index('\n') + 1:]
+            self.bmetadata = json.loads(self.getHeader('meta'))
+            self.btype = self.getMetadata('type')
+            self.powHash = self.getMetadata('powHash')
+            self.powToken = self.getMetadata('powToken')
+            self.signed = ('sig' in self.getHeader() and self.getHeader('sig') != '')
+            self.signature = (None if not self.isSigned() else self.getHeader('sig'))
+            self.signedData = (None if not self.isSigned() else self.getHeader('meta') + '\n' + self.getContent())
+            self.date = self.getCore().getBlockDate(self.getHash())
+
+            if not self.getDate() is None:
+                self.date = datetime.datetime.fromtimestamp(self.getDate())
+
+            self.valid = True
+            return True
+        except Exception as e:
+            logger.error('Failed to update block data.', error = e, timestamp = False)
+
+        self.valid = False
+        return False
+
+    def delete(self):
+        '''
+            Deletes the block's file and records, if they exist
+
+            Outputs:
+            - (bool): whether or not the operation was successful
+        '''
+
+        if self.exists():
+            os.remove(self.getBlockFile())
+            removeBlock(self.getHash())
+            return True
+        return False
+
+    def save(self, sign = False, recreate = True):
+        '''
+            Saves a block to file and imports it into Onionr
+
+            Inputs:
+            - sign (bool): whether or not to sign the block before saving
+            - recreate (bool): if the block already exists, whether or not to recreate the block and save under a new hash
+
+            Outputs:
+            - (bool): whether or not the operation was successful
+        '''
+
+        try:
+            if self.isValid() is True:
+                if (not self.getBlockFile() is None) and (recreate is True):
+                    with open(self.getBlockFile(), 'wb') as blockFile:
+                        blockFile.write(self.getRaw().encode())
+                    self.update()
+                else:
+                    self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign)
+                    self.update()
+                return True
+            else:
+                logger.warn('Not writing block; it is invalid.')
+        except Exception as e:
+            logger.error('Failed to save block.', error = e, timestamp = False)
+            return False
+
+    # getters
+
+    def getHash(self):
+        '''
+            Returns the hash of the block if saved to file
+
+            Outputs:
+            - (str): the hash of the block, or None
+        '''
+
+        return self.hash
+
+    def getCore(self):
+        '''
+            Returns the Core instance being used by the Block
+
+            Outputs:
+            - (Core): the Core instance
+        '''
+
+        return self.core
+
+    def getType(self):
+        '''
+            Returns the type of the block
+
+            Outputs:
+            - (str): the type of the block
+        '''
+
+        return self.btype
+
+    def getRaw(self):
+        '''
+            Returns the raw contents of the block, if saved to file
+
+            Outputs:
+            - (str): the raw contents of the block, or None
+        '''
+
+        return str(self.raw)
+
+    def getHeader(self, key = None):
+        '''
+            Returns the header information
+
+            Inputs:
+            - key (str): only returns the value of the key in the header
+
+            Outputs:
+            - (dict/str): either the whole header as a dict, or one value
+        '''
+
+        if not key is None:
+            return self.getHeader()[key]
+        else:
+            return self.bheader
+
+    def getMetadata(self, key = None):
+        '''
+            Returns the metadata information
+
+            Inputs:
+            - key (str): only returns the value of the key in the metadata
+
+            Outputs:
+            - (dict/str): either the whole metadata as a dict, or one value
+        '''
+
+        if not key is None:
+            return self.getMetadata()[key]
+        else:
+            return self.bmetadata
+
+    def getContent(self):
+        '''
+            Returns the contents of the block
+
+            Outputs:
+            - (str): the contents of the block
+        '''
+
+        return str(self.bcontent)
+
+    def getDate(self):
+        '''
+            Returns the date that the block was received, if loaded from file
+
+            Outputs:
+            - (datetime): the date that the block was received
+        '''
+
+        return self.date
+
+    def getBlockFile(self):
+        '''
+            Returns the location of the block file if it is saved
+
+            Outputs:
+            - (str): the location of the block file, or None
+        '''
+
+        return self.blockFile
+
+    def isValid(self):
+        '''
+            Checks if the block is valid
+
+            Outputs:
+            - (bool): whether or not the block is valid
+        '''
+
+        return self.valid
+
+    def isSigned(self):
+        '''
+            Checks if the block was signed
+
+            Outputs:
+            - (bool): whether or not the block is signed
+        '''
+
+        return self.signed
+
+    def getSignature(self):
+        '''
+            Returns the base64-encoded signature
+
+            Outputs:
+            - (str): the signature, or None
+        '''
+
+        return self.signature
+
+    def getSignedData(self):
+        '''
+            Returns the data that was signed
+
+            Outputs:
+            - (str): the data that was signed, or None
+        '''
+
+        return self.signedData
+
+    def isSigner(self, signer, encodedData = True):
+        '''
+            Checks if the block was signed by the signer inputted
+
+            Inputs:
+            - signer (str): the public key of the signer to check against
+            - encodedData (bool): whether or not the `signer` argument is base64 encoded
+
+            Outputs:
+            - (bool): whether or not the signer of the block is the signer inputted
+        '''
+
+        try:
+            if (not self.isSigned()) or (not self.getCore()._utils.validatePubKey(signer)):
+                return False
+
+            return bool(self.getCore()._crypto.edVerify(self.getSignedData(), signer, self.getSignature(), encodedData = encodedData))
+        except:
+            return False
+
+    # setters
+
+    def setType(self, btype):
+        '''
+            Sets the type of the block
+
+            Inputs:
+            - btype (str): the type of block to be set to
+
+            Outputs:
+            - (Block): the block instance
+        '''
+
+        self.btype = btype
+        return self
+
+    def setContent(self, bcontent):
+        '''
+            Sets the contents of the block
+
+            Inputs:
+            - bcontent (str): the contents to be set to
+
+            Outputs:
+            - (Block): the block instance
+        '''
+
+        self.bcontent = str(bcontent)
+        return self
+
+    # static
+
+    def getBlocks(type = None, signer = None, signed = None, reverse = False, core = None):
+        '''
+            Returns a list of Block objects based on supplied filters
+
+            Inputs:
+            - type (str): filters by block type
+            - signer (str/list): filters by signer (one in the list has to be a signer)
+            - signed (bool): filters out by whether or not the block is signed
+            - reverse (bool): reverses the list if True
+            - core (Core): lets you optionally supply a core instance so one doesn't need to be started
+
+            Outputs:
+            - (list): a list of Block objects that match the input
+        '''
+
+        try:
+            core = (core if not core is None else onionrcore.Core())
+
+            relevant_blocks = list()
+            blocks = (core.getBlockList() if type is None else core.getBlocksByType(type))
+
+            for block in blocks:
+                if Block.exists(block):
+                    block = Block(block, core = core)
+
+                    relevant = True
+
+                    if (not signed is None) and (block.isSigned() != bool(signed)):
+                        relevant = False
+                    if not signer is None:
+                        if isinstance(signer, (str,)):
+                            signer = [signer]
+
+                        isSigner = False
+                        for key in signer:
+                            if block.isSigner(key):
+                                isSigner = True
+                                break
+
+                        if not isSigner:
+                            relevant = False
+
+                    if relevant:
+                        relevant_blocks.append(block)
+
+            if bool(reverse):
+                relevant_blocks.reverse()
+
+            return relevant_blocks
+        except Exception as e:
+            logger.debug(('Failed to get blocks: %s' % str(e)) + logger.parse_error())
+
+        return list()
+
+    def exists(hash):
+        '''
+            Checks if a block is saved to file or not
+
+            Inputs:
+            - hash (str/Block):
+              - if (Block): check if this block is saved to file
+              - if (str): check if a block by this hash is in file
+
+            Outputs:
+            - (bool): whether or not the block file exists
+        '''
+
+        if hash is None:
+            return False
+        elif type(hash) == Block:
+            blockfile = hash.getBlockFile()
+        else:
+            blockfile = 'data/blocks/%s.dat' % hash
+
+        return os.path.exists(blockfile) and os.path.isfile(blockfile)

onionr/onionrcrypto.py

@@ -17,47 +17,89 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import nacl.signing, nacl.encoding, nacl.public, nacl.secret, os, binascii, base64
+import nacl.signing, nacl.encoding, nacl.public, nacl.hash, nacl.secret, os, binascii, base64, hashlib, logger, onionrproofs, time, math
 
 class OnionrCrypto:
     def __init__(self, coreInstance):
         self._core = coreInstance
         self._keyFile = 'data/keys.txt'
+        self.keyPowFile = 'data/keyPow.txt'
         self.pubKey = None
         self.privKey = None
 
+        self.pubKeyPowToken = None
+        #self.pubKeyPowHash = None
+
+        self.HASH_ID_ROUNDS = 2000
+
         # Load our own pub/priv Ed25519 keys, gen & save them if they don't exist
         if os.path.exists(self._keyFile):
             with open('data/keys.txt', 'r') as keys:
                 keys = keys.read().split(',')
                 self.pubKey = keys[0]
                 self.privKey = keys[1]
+            try:
+                with open(self.keyPowFile, 'r') as powFile:
+                    data = powFile.read()
+                    self.pubKeyPowToken = data
+            except (FileNotFoundError, IndexError):
+                pass
         else:
             keys = self.generatePubKey()
             self.pubKey = keys[0]
             self.privKey = keys[1]
             with open(self._keyFile, 'w') as keyfile:
                 keyfile.write(self.pubKey + ',' + self.privKey)
+            with open(self.keyPowFile, 'w') as keyPowFile:
+                proof = onionrproofs.POW(self.pubKey)
+                logger.info('Doing necessary work to insert our public key')
+                while True:
+                    time.sleep(0.2)
+                    powToken = proof.getResult()
+                    if powToken != False:
+                        break
+                keyPowFile.write(base64.b64encode(powToken[1]).decode())
+                self.pubKeyPowToken = powToken[1]
+                self.pubKeyPowHash = powToken[0]
         return
 
-    def edVerify(self, data, key):
+    def edVerify(self, data, key, sig, encodedData=True):
         '''Verify signed data (combined in nacl) to an ed25519 key'''
+        try:
             key = nacl.signing.VerifyKey(key=key, encoder=nacl.encoding.Base32Encoder)
-        retData = ''
+        except nacl.exceptions.ValueError:
-        if encodeResult:
+            logger.warn('Signature by unknown key (cannot reverse hash)')
-            retData = key.verify(data.encode(), encoder=nacl.encoding.Base64Encoder) # .encode() is not the same as nacl.encoding
+            return False
+        retData = False
+        sig = base64.b64decode(sig)
+        try:
+            data = data.encode()
+        except AttributeError:
+            pass
+        if encodedData:
+            try:
+                retData = key.verify(data, sig) # .encode() is not the same as nacl.encoding
+            except nacl.exceptions.BadSignatureError:
+                pass
         else:
-            retData = key.verify(data.encode())
+            try:
+                retData = key.verify(data, sig)
+            except nacl.exceptions.BadSignatureError:
+                pass
         return retData
 
     def edSign(self, data, key, encodeResult=False):
         '''Ed25519 sign data'''
+        try:
+            data = data.encode()
+        except AttributeError:
+            pass
         key = nacl.signing.SigningKey(seed=key, encoder=nacl.encoding.Base32Encoder)
         retData = ''
         if encodeResult:
-            retData = key.sign(data.encode(), encoder=nacl.encoding.Base64Encoder) # .encode() is not the same as nacl.encoding
+            retData = key.sign(data, encoder=nacl.encoding.Base64Encoder).signature.decode() # .encode() is not the same as nacl.encoding
         else:
-            retData = key.sign(data.encode())
+            retData = key.sign(data).signature
         return retData
 
     def pubKeyEncrypt(self, data, pubkey, anonymous=False, encodedData=False):
@@ -70,7 +112,7 @@ class OnionrCrypto:
             encoding = nacl.encoding.RawEncoder
 
         if self.privKey != None and not anonymous:
-            ownKey = nacl.signing.SigningKey(seed=self.privKey, encoder=nacl.encoding.Base32Encoder())
+            ownKey = nacl.signing.SigningKey(seed=self.privKey, encoder=nacl.encoding.Base32Encoder)
             key = nacl.signing.VerifyKey(key=pubkey, encoder=nacl.encoding.Base32Encoder).to_curve25519_public_key()
             ourBox = nacl.public.Box(ownKey, key)
             retVal = ourBox.encrypt(data.encode(), encoder=encoding)
@@ -80,20 +122,20 @@ class OnionrCrypto:
             retVal = anonBox.encrypt(data.encode(), encoder=encoding)
         return retVal
 
-    def pubKeyDecrypt(self, data, pubkey, anonymous=False, encodedData=False):
+    def pubKeyDecrypt(self, data, pubkey='', anonymous=False, encodedData=False):
         '''pubkey decrypt (Curve25519, taken from Ed25519 pubkey)'''
-        retVal = ''
+        retVal = False
         if encodedData:
             encoding = nacl.encoding.Base64Encoder
         else:
             encoding = nacl.encoding.RawEncoder
-        ownKey = nacl.signing.SigningKey(seed=self.privKey, encoder=nacl.encoding.Base32Encoder())
+        ownKey = nacl.signing.SigningKey(seed=self.privKey, encoder=nacl.encoding.Base32Encoder()).to_curve25519_private_key()
-        if self.privKey != None and not anoymous:
+        if self.privKey != None and not anonymous:
             ourBox = nacl.public.Box(ownKey, pubkey)
             decrypted = ourBox.decrypt(data, encoder=encoding)
         elif anonymous:
             anonBox = nacl.public.SealedBox(ownKey)
-            decrypted = anonBox.decrypt(data.encode(), encoder=encoding)
+            decrypted = anonBox.decrypt(data, encoder=encoding)
         return decrypted
 
     def symmetricPeerEncrypt(self, data, peer):
@@ -116,8 +158,6 @@ class OnionrCrypto:
         decrypted = self.symmetricDecrypt(data, key, encodedKey=True)
         return decrypted
 
-        return
-
     def symmetricEncrypt(self, data, key, encodedKey=False, returnEncoded=True):
         '''Encrypt data to a 32-byte key (Salsa20-Poly1305 MAC)'''
         if encodedKey:
@@ -167,7 +207,69 @@ class OnionrCrypto:
         return binascii.hexlify(nacl.utils.random(nacl.secret.SecretBox.KEY_SIZE))
 
     def generatePubKey(self):
-        '''Generate a Ed25519 public key pair, return tuple of base64encoded pubkey, privkey'''
+        '''Generate a Ed25519 public key pair, return tuple of base32encoded pubkey, privkey'''
         private_key = nacl.signing.SigningKey.generate()
         public_key = private_key.verify_key.encode(encoder=nacl.encoding.Base32Encoder())
         return (public_key.decode(), private_key.encode(encoder=nacl.encoding.Base32Encoder()).decode())
+
+    def pubKeyHashID(self, pubkey=''):
+        '''Accept a ed25519 public key, return a truncated result of X many sha3_256 hash rounds'''
+        if pubkey == '':
+            pubkey = self.pubKey
+        prev = ''
+        pubkey = pubkey.encode()
+        for i in range(self.HASH_ID_ROUNDS):
+            try:
+                prev = prev.encode()
+            except AttributeError:
+                pass
+            hasher = hashlib.sha3_256()
+            hasher.update(pubkey + prev)
+            prev = hasher.hexdigest()
+        result = prev
+        return result
+
+    def sha3Hash(self, data):
+        hasher = hashlib.sha3_256()
+        hasher.update(data)
+        return hasher.hexdigest()
+
+    def blake2bHash(self, data):
+        try:
+            data = data.encode()
+        except AttributeError:
+            pass
+        return nacl.hash.blake2b(data)
+
+    def verifyPow(self, blockContent, metadata):
+        '''
+            Verifies the proof of work associated with a block
+        '''
+        retData = False
+
+        if not (('powToken' in metadata) and ('powHash' in metadata)):
+            return False
+
+        dataLen = len(blockContent)
+
+        expectedHash = self.blake2bHash(base64.b64decode(metadata['powToken']) + self.blake2bHash(blockContent.encode()))
+        difficulty = 0
+        try:
+            expectedHash = expectedHash.decode()
+        except AttributeError:
+            pass
+        if metadata['powHash'] == expectedHash:
+            difficulty = math.floor(dataLen / 1000000)
+
+            mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
+            puzzle = mainHash[:difficulty]
+
+            if metadata['powHash'][:difficulty] == puzzle:
+                # logger.debug('Validated block pow')
+                retData = True
+            else:
+                logger.debug("Invalid token (#1)")
+        else:
+            logger.debug('Invalid token (#2): Expected hash %s, got hash %s...' % (metadata['powHash'], expectedHash))
+
+        return retData

onionr/onionrevents.py

@@ -18,20 +18,41 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 
-import config, logger, onionrplugins as plugins
+import config, logger, onionrplugins as plugins, onionrpluginapi as pluginapi
+from threading import Thread
 
-def event(event_name, data = None, onionr = None):
+def get_pluginapi(onionr, data):
+    return pluginapi.pluginapi(onionr, data)
+
+def __event_caller(event_name, data = {}, onionr = None):
+    '''
+        DO NOT call this function, this is for threading code only.
+        Instead, call onionrevents.event
+    '''
+    for plugin in plugins.get_enabled_plugins():
+        try:
+            call(plugins.get_plugin(plugin), event_name, data, get_pluginapi(onionr, data))
+        except ModuleNotFoundError as e:
+            logger.warn('Disabling nonexistant plugin \"' + plugin + '\"...')
+            plugins.disable(plugin, onionr, stop_event = False)
+        except Exception as e:
+            logger.warn('Event \"' + event_name + '\" failed for plugin \"' + plugin + '\".')
+            logger.debug(str(e))
+
+
+def event(event_name, data = {}, onionr = None, threaded = True):
     '''
         Calls an event on all plugins (if defined)
     '''
 
-    for plugin in plugins.get_enabled_plugins():
+    if threaded:
-        try:
+        thread = Thread(target = __event_caller, args = (event_name, data, onionr))
-            call(plugins.get_plugin(plugin), event_name, data, onionr)
+        thread.start()
-        except:
+        return thread
-            logger.warn('Event \"' + event_name + '\" failed for plugin \"' + plugin + '\".')
+    else:
+        __event_caller(event_name, data, onionr)
 
-def call(plugin, event_name, data = None, onionr = None):
+def call(plugin, event_name, data = None, pluginapi = None):
     '''
         Calls an event on a plugin if one is defined
     '''
@@ -42,12 +63,12 @@ def call(plugin, event_name, data = None, onionr = None):
 
             # TODO: Use multithreading perhaps?
             if hasattr(plugin, attribute):
-                logger.debug('Calling event ' + str(event_name))
+                #logger.debug('Calling event ' + str(event_name))
-                getattr(plugin, attribute)(onionr, data)
+                getattr(plugin, attribute)(pluginapi)
 
             return True
-        except:
+        except Exception as e:
-            logger.warn('Failed to call event ' + str(event_name) + ' on module.')
+            logger.debug(str(e))
             return False
     else:
         return True

onionr/onionri2p.py

@@ -0,0 +1,19 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network
+
+    Funcitons for talking to I2P
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''

onionr/onionrpluginapi.py

@@ -0,0 +1,171 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network
+
+    This file deals with the object that is passed with each event
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import onionrplugins, core as onionrcore, logger
+
+class DaemonAPI:
+    def __init__(self, pluginapi):
+        self.pluginapi = pluginapi
+
+    def start(self):
+        self.pluginapi.get_onionr().daemon()
+
+        return
+
+    def stop(self):
+        self.pluginapi.get_onionr().killDaemon()
+
+        return
+
+    def queue(self, command, data = ''):
+        self.pluginapi.get_core().daemonQueueAdd(command, data)
+
+        return
+
+    def local_command(self, command):
+        return self.pluginapi.get_utils().localCommand(self, command)
+
+    def queue_pop(self):
+        return self.get_core().daemonQueue()
+
+class PluginAPI:
+    def __init__(self, pluginapi):
+        self.pluginapi = pluginapi
+
+    def start(self, name):
+        onionrplugins.start(name)
+
+    def stop(self, name):
+        onionrplugins.stop(name)
+
+    def reload(self, name):
+        onionrplugins.reload(name)
+
+    def enable(self, name):
+        onionrplugins.enable(name)
+
+    def disable(self, name):
+        onionrplugins.disable(name)
+
+    def event(self, name, data = {}):
+        events.event(name, data = data, onionr = self.pluginapi.get_onionr())
+
+    def is_enabled(self, name):
+        return onionrplugins.is_enabled(name)
+
+    def get_enabled_plugins(self):
+        return onionrplugins.get_enabled()
+
+    def get_folder(self, name = None, absolute = True):
+        return onionrplugins.get_plugins_folder(name = name, absolute = absolute)
+
+    def get_data_folder(self, name, absolute = True):
+        return onionrplugins.get_plugin_data_folder(name, absolute = absolute)
+
+    def daemon_event(self, event, plugin = None):
+        return # later make local command like /client/?action=makeEvent&event=eventname&module=modulename
+
+class CommandAPI:
+    def __init__(self, pluginapi):
+        self.pluginapi = pluginapi
+
+    def register(self, names, call = None):
+        if isinstance(names, str):
+            names = [names]
+
+        for name in names:
+            self.pluginapi.get_onionr().addCommand(name, call)
+
+        return
+
+    def unregister(self, names):
+        if isinstance(names, str):
+            names = [names]
+
+        for name in names:
+            self.pluginapi.get_onionr().delCommand(name)
+
+        return
+
+    def register_help(self, names, description):
+        if isinstance(names, str):
+            names = [names]
+
+        for name in names:
+            self.pluginapi.get_onionr().addHelp(name, description)
+
+        return
+
+    def unregister_help(self, names):
+        if isinstance(names, str):
+            names = [names]
+
+        for name in names:
+            self.pluginapi.get_onionr().delHelp(name)
+
+        return
+
+    def call(self, name):
+        self.pluginapi.get_onionr().execute(name)
+
+        return
+
+    def get_commands(self):
+        return self.pluginapi.get_onionr().getCommands()
+
+class pluginapi:
+    def __init__(self, onionr, data):
+        self.onionr = onionr
+        self.data = data
+        if self.onionr is None:
+            self.core = onionrcore.Core()
+        else:
+            self.core = self.onionr.onionrCore
+
+        self.daemon = DaemonAPI(self)
+        self.plugins = PluginAPI(self)
+        self.commands = CommandAPI(self)
+
+    def get_onionr(self):
+        return self.onionr
+
+    def get_data(self):
+        return self.data
+
+    def get_core(self):
+        return self.core
+
+    def get_utils(self):
+        return self.get_core()._utils
+
+    def get_crypto(self):
+        return self.get_core()._crypto
+
+    def get_daemonapi(self):
+        return self.daemon
+
+    def get_pluginapi(self):
+        return self.plugins
+
+    def get_commandapi(self):
+        return self.commands
+
+    def is_development_mode(self):
+        return self.get_onionr()._developmentMode

onionr/onionrplugins.py

@@ -24,7 +24,7 @@ import onionrevents as events
 _pluginsfolder = 'data/plugins/'
 _instances = dict()
 
-def reload(stop_event = True):
+def reload(onionr = None, stop_event = True):
     '''
         Reloads all the plugins
     '''
@@ -41,10 +41,10 @@ def reload(stop_event = True):
 
         if stop_event is True:
             for plugin in enabled_plugins:
-                stop(plugin)
+                stop(plugin, onionr)
 
         for plugin in enabled_plugins:
-            start(plugin)
+            start(plugin, onionr)
 
         return True
     except:
@@ -53,7 +53,7 @@ def reload(stop_event = True):
     return False
 
 
-def enable(name, start_event = True):
+def enable(name, onionr = None, start_event = True):
     '''
         Enables a plugin
     '''
@@ -62,17 +62,20 @@ def enable(name, start_event = True):
 
     if exists(name):
         enabled_plugins = get_enabled_plugins()
+        if not name in enabled_plugins:
             enabled_plugins.append(name)
             config_plugins = config.get('plugins')
             config_plugins['enabled'] = enabled_plugins
             config.set('plugins', config_plugins, True)
 
-        events.call(get_plugin(name), 'enable')
+            events.call(get_plugin(name), 'enable', onionr)
 
             if start_event is True:
                 start(name)
 
             return True
+        else:
+            return False
     else:
         logger.error('Failed to enable plugin \"' + name + '\", disabling plugin.')
         disable(name)
@@ -80,7 +83,7 @@ def enable(name, start_event = True):
         return False
 
 
-def disable(name, stop_event = True):
+def disable(name, onionr = None, stop_event = True):
     '''
         Disables a plugin
     '''
@@ -95,12 +98,12 @@ def disable(name, stop_event = True):
         config.set('plugins', config_plugins, True)
 
     if exists(name):
-        events.call(get_plugin(name), 'disable')
+        events.call(get_plugin(name), 'disable', onionr)
 
         if stop_event is True:
             stop(name)
 
-def start(name):
+def start(name, onionr = None):
     '''
         Starts the plugin
     '''
@@ -114,7 +117,7 @@ def start(name):
             if plugin is None:
                 raise Exception('Failed to import module.')
             else:
-                events.call(plugin, 'start')
+                events.call(plugin, 'start', onionr)
 
             return plugin
         except:
@@ -124,7 +127,7 @@ def start(name):
 
     return None
 
-def stop(name):
+def stop(name, onionr = None):
     '''
         Stops the plugin
     '''
@@ -138,7 +141,7 @@ def stop(name):
             if plugin is None:
                 raise Exception('Failed to import module.')
             else:
-                events.call(plugin, 'stop')
+                events.call(plugin, 'stop', onionr)
 
             return plugin
         except:
@@ -204,12 +207,19 @@ def get_plugins_folder(name = None, absolute = True):
         path = _pluginsfolder
     else:
         # only allow alphanumeric characters
-        path = _pluginsfolder + re.sub('[^0-9a-zA-Z]+', '', str(name).lower()) + '/'
+        path = _pluginsfolder + re.sub('[^0-9a-zA-Z]+', '', str(name).lower())
 
     if absolute is True:
         path = os.path.abspath(path)
 
-    return path
+    return path + '/'
+
+def get_plugin_data_folder(name, absolute = True):
+    '''
+        Returns the location of a plugin's data folder
+    '''
+
+    return get_plugins_folder(name, absolute) + 'data/'
 
 def check():
     '''
@@ -226,9 +236,4 @@ def check():
         logger.debug('Generating plugin data folder...')
         os.makedirs(os.path.dirname(get_plugins_folder()))
 
-    if not exists('test'):
-        os.makedirs(get_plugins_folder('test'))
-        with open(get_plugins_folder('test') + '/main.py', 'a') as main:
-            main.write("print('Running')\n\ndef on_test(onionr = None, data = None):\n    print('received test event!')\n    return True\n\ndef on_start(onionr = None, data = None):\n    print('start event called')\n\ndef on_stop(onionr = None, data = None):\n    print('stop event called')\n\ndef on_enable(onionr = None, data = None):\n    print('enable event called')\n\ndef on_disable(onionr = None, data = None):\n    print('disable event called')\n")
-        enable('test')
     return

onionr/onionrproofs.py

@@ -17,10 +17,12 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger
+
-import btc
+import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys
+import core
+
 class POW:
-    def pow(self, reporting=False):
+    def pow(self, reporting = False):
         startTime = math.floor(time.time())
         self.hashing = True
         self.reporting = reporting
@@ -28,42 +30,48 @@ class POW:
         answer = ''
         heartbeat = 200000
         hbCount = 0
-        blockCheck = 300000 # How often the hasher should check if the bitcoin block is updated (slows hashing but prevents less wasted work)
+        myCore = core.Core()
-        blockCheckCount = 0
-        block = ''#self.bitcoinNode.getBlockHash(self.bitcoinNode.getLastBlockHeight())
         while self.hashing:
-            if blockCheckCount == blockCheck:
+            rand = nacl.utils.random()
-                if self.reporting:
+            token = nacl.hash.blake2b(rand + self.data).decode()
-                    logger.debug('Refreshing Bitcoin block')
+            #print(token)
-                block = ''#self.bitcoinNode.getBlockHash(self.bitcoinNode.getLastBlockHeight())
+            if self.puzzle == token[0:self.difficulty]:
-                blockCheckCount = 0
-            blockCheckCount += 1
-            hbCount += 1
-            token = nacl.hash.blake2b(nacl.utils.random() + block.encode()).decode()
-            if self.mainHash[0:self.difficulty] == token[0:self.difficulty]:
                 self.hashing = False
                 iFound = True
                 break
+        else:
+            logger.debug('POW thread exiting, another thread found result')
         if iFound:
             endTime = math.floor(time.time())
             if self.reporting:
-                logger.info('Found token ' + token)
+                logger.info('Found token ' + token, timestamp=True)
-                logger.info('took ' + str(endTime - startTime))
+                logger.info('took ' + str(endTime - startTime) + ' seconds', timestamp=True)
-            self.result = token
+            self.result = (token, rand)
 
-    def __init__(self, difficulty, bitcoinNode):
+    def __init__(self, data):
         self.foundHash = False
-        self.difficulty = difficulty
+        self.difficulty = 0
+        self.data = data
+
+        dataLen = sys.getsizeof(data)
+        self.difficulty = math.floor(dataLen/1000000)
+        if self.difficulty <= 2:
+            self.difficulty = 4
+
+        try:
+            self.data = self.data.encode()
+        except AttributeError:
+            pass
+        self.data = nacl.hash.blake2b(self.data)
 
         logger.debug('Computing difficulty of ' + str(self.difficulty))
 
-        self.mainHash = nacl.hash.blake2b(nacl.utils.random()).decode()
+        self.mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
         self.puzzle = self.mainHash[0:self.difficulty]
-        self.bitcoinNode = bitcoinNode
+        #logger.debug('trying to find ' + str(self.mainHash))
-        logger.debug('trying to find ' + str(self.mainHash))
         tOne = threading.Thread(name='one', target=self.pow, args=(True,))
-        tTwo = threading.Thread(name='two', target=self.pow)
+        tTwo = threading.Thread(name='two', target=self.pow, args=(True,))
-        tThree = threading.Thread(name='three', target=self.pow)
+        tThree = threading.Thread(name='three', target=self.pow, args=(True,))
         tOne.start()
         tTwo.start()
         tThree.start()
@@ -77,7 +85,9 @@ class POW:
         self.difficulty = newDiff
 
     def getResult(self):
-        '''Returns the result then sets to false, useful to automatically clear the result'''
+        '''
+            Returns the result then sets to false, useful to automatically clear the result
+        '''
         try:
             retVal = self.result
         except AttributeError:

onionr/onionrutils.py

@@ -18,7 +18,7 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
 # Misc functions that do not fit in the main api, but are useful
-import getpass, sys, requests, os, socket, hashlib, logger, sqlite3, config
+import getpass, sys, requests, os, socket, hashlib, logger, sqlite3, config, binascii, time, base64, json, glob, shutil, math
 import nacl.signing, nacl.encoding
 
 if sys.version_info < (3, 6):
@@ -35,56 +35,141 @@ class OnionrUtils:
     def __init__(self, coreInstance):
         self.fingerprintFile = 'data/own-fingerprint.txt'
         self._core = coreInstance
+
+        self.timingToken = ''
+
         return
 
-    def sendPM(self, user, message):
+    def getTimeBypassToken(self):
-        '''High level function to encrypt a message to a peer and insert it as a block'''
+        try:
+            if os.path.exists('data/time-bypass.txt'):
+                with open('data/time-bypass.txt', 'r') as bypass:
+                    self.timingToken = bypass.read()
+        except Exception as error:
+            logger.error('Failed to fetch time bypass token.', error=error)
+
+    def sendPM(self, pubkey, message):
+        '''
+            High level function to encrypt a message to a peer and insert it as a block
+        '''
+
+        try:
+            # We sign PMs here rather than in core.insertBlock in order to mask the sender's pubkey
+            payload = {'sig': '', 'msg': '', 'id': self._core._crypto.pubKey}
+
+            sign = self._core._crypto.edSign(message, self._core._crypto.privKey, encodeResult=True)
+            #encrypted = self._core._crypto.pubKeyEncrypt(message, pubkey, anonymous=True, encodedData=True).decode()
+
+            payload['sig'] = sign
+            payload['msg'] = message
+            payload = json.dumps(payload)
+            message = payload
+            encrypted = self._core._crypto.pubKeyEncrypt(message, pubkey, anonymous=True, encodedData=True).decode()
+
+
+            block = self._core.insertBlock(encrypted, header='pm', sign=False)
+            if block == '':
+                logger.error('Could not send PM')
+            else:
+                logger.info('Sent PM, hash: %s' % block)
+        except Exception as error:
+            logger.error('Failed to send PM.', error=error)
+
         return
 
     def incrementAddressSuccess(self, address):
-        '''Increase the recorded sucesses for an address'''
+        '''
+            Increase the recorded sucesses for an address
+        '''
         increment = self._core.getAddressInfo(address, 'success') + 1
         self._core.setAddressInfo(address, 'success', increment)
         return
 
     def decrementAddressSuccess(self, address):
-        '''Decrease the recorded sucesses for an address'''
+        '''
+            Decrease the recorded sucesses for an address
+        '''
         increment = self._core.getAddressInfo(address, 'success') - 1
         self._core.setAddressInfo(address, 'success', increment)
         return
 
     def mergeKeys(self, newKeyList):
-        '''Merge ed25519 key list to our database'''
+        '''
+            Merge ed25519 key list to our database
+        '''
+        try:
             retVal = False
             if newKeyList != False:
-            for key in newKeyList:
+                for key in newKeyList.split(','):
-                if not key in self._core.listPeers(randomOrder=False):
+                    key = key.split('-')
-                    if self._core.addPeer(key):
+                    try:
+                        if len(key[0]) > 60 or len(key[1]) > 1000:
+                            logger.warn('%s or its pow value is too large.' % key[0])
+                            continue
+                    except IndexError:
+                        logger.warn('No pow token')
+                        continue
+                    powHash = self._core._crypto.blake2bHash(base64.b64decode(key[1]) + self._core._crypto.blake2bHash(key[0].encode()))
+                    try:
+                        powHash = powHash.encode()
+                    except AttributeError:
+                        pass
+                    if powHash.startswith(b'0000'):
+                        if not key[0] in self._core.listPeers(randomOrder=False) and type(key) != None and key[0] != self._core._crypto.pubKey:
+                            if self._core.addPeer(key[0], key[1]):
                                 retVal = True
+                    else:
+                        logger.warn(powHash)
+                        logger.warn('%s pow failed' % key[0])
             return retVal
+        except Exception as error:
+            logger.error('Failed to merge keys.', error=error)
+            return False
 
 
     def mergeAdders(self, newAdderList):
-        '''Merge peer adders list to our database'''
+        '''
+            Merge peer adders list to our database
+        '''
+        try:
             retVal = False
             if newAdderList != False:
-            for adder in newAdderList:
+                for adder in newAdderList.split(','):
-                if not adder in self._core.listAdders(randomOrder=False):
+                    if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress():
                         if self._core.addAddress(adder):
+                            logger.info('Added %s to db.' % adder, timestamp = True)
                             retVal = True
+                    else:
+                        logger.debug('%s is either our address or already in our DB' % adder)
             return retVal
+        except Exception as error:
+            logger.error('Failed to merge adders.', error = error)
+            return False
 
-    def localCommand(self, command):
+    def getMyAddress(self):
+        try:
+            with open('./data/hs/hostname', 'r') as hostname:
+                return hostname.read().strip()
+        except Exception as error:
+            logger.error('Failed to read my address.', error = error)
+            return None
+
+    def localCommand(self, command, silent = True):
         '''
             Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers.
         '''
 
         config.reload()
-
+        self.getTimeBypassToken()
         # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
-        requests.get('http://' + open('data/host.txt', 'r').read() + ':' + str(config.get('client')['port']) + '/client/?action=' + command + '&token=' + str(config.get('client')['client_hmac']))
+        try:
+            retData = requests.get('http://' + open('data/host.txt', 'r').read() + ':' + str(config.get('client')['port']) + '/client/?action=' + command + '&token=' + str(config.get('client')['client_hmac']) + '&timingToken=' + self.timingToken).text
+        except Exception as error:
+            if not silent:
+                logger.error('Failed to make local request (command: %s).' % command, error=error)
+            retData = False
 
-        return
+        return retData
 
     def getPassword(self, message='Enter password: ', confirm = True):
         '''
@@ -139,6 +224,7 @@ class OnionrUtils:
         '''
             Return a sha3_256 hash of the blocks DB
         '''
+        try:
             with open(self._core.blockDB, 'rb') as data:
                 data = data.read()
             hasher = hashlib.sha3_256()
@@ -146,6 +232,8 @@ class OnionrUtils:
             dataHash = hasher.hexdigest()
 
             return dataHash
+        except Exception as error:
+            logger.error('Failed to get block DB hash.', error=error)
 
     def hasBlock(self, hash):
         '''
@@ -165,6 +253,12 @@ class OnionrUtils:
                 conn.close()
                 return False
 
+    def hasKey(self, key):
+        '''
+            Check for key in list of public keys
+        '''
+        return key in self._core.listPeers()
+
     def validateHash(self, data, length=64):
         '''
             Validate if a string is a valid hex formatted hash
@@ -184,12 +278,16 @@ class OnionrUtils:
         return retVal
 
     def validatePubKey(self, key):
-        '''Validate if a string is a valid base32 encoded Ed25519 key'''
+        '''
+            Validate if a string is a valid base32 encoded Ed25519 key
+        '''
         retVal = False
         try:
             nacl.signing.SigningKey(seed=key, encoder=nacl.encoding.Base32Encoder)
         except nacl.exceptions.ValueError:
             pass
+        except base64.binascii.Error as err:
+            pass
         else:
             retVal = True
         return retVal
@@ -199,6 +297,7 @@ class OnionrUtils:
         '''
             Validate if an address is a valid tor or i2p hidden service
         '''
+        try:
             idLength = len(id)
             retVal = True
             idNoDomain = ''
@@ -238,3 +337,168 @@ class OnionrUtils:
                     retVal = False
 
             return retVal
+        except:
+            return False
+
+    def loadPMs(self):
+        '''
+            Find, decrypt, and return array of PMs (array of dictionary, {from, text})
+        '''
+        #blocks = self._core.getBlockList()
+        blocks = self._core.getBlocksByType('pm')
+        message = ''
+        sender = ''
+        for i in blocks:
+            if len (i) == 0:
+                continue
+            try:
+                with open('data/blocks/' + i + '.dat', 'r') as potentialMessage:
+                    potentialMessage = potentialMessage.read()
+                    blockMetadata = json.loads(potentialMessage[:potentialMessage.find('\n')])
+                    blockContent = potentialMessage[potentialMessage.find('\n') + 1:]
+
+                    try:
+                        message = self._core._crypto.pubKeyDecrypt(blockContent, encodedData=True, anonymous=True)
+                    except nacl.exceptions.CryptoError as e:
+                        pass
+                    else:
+                        try:
+                            message = message.decode()
+                        except AttributeError:
+                            pass
+
+                        try:
+                            message = json.loads(message)
+                        except json.decoder.JSONDecodeError:
+                            pass
+                        else:
+                            logger.info('Decrypted %s:' % i)
+                            logger.info(message["msg"])
+
+                            signer = message["id"]
+                            sig = message["sig"]
+
+                            if self.validatePubKey(signer):
+                                if self._core._crypto.edVerify(message["msg"], signer, sig, encodedData=True):
+                                    logger.info("Good signature by %s" % signer)
+                                else:
+                                    logger.warn("Bad signature by %s" % signer)
+                            else:
+                                logger.warn('Bad sender id: %s' % signer)
+
+            except FileNotFoundError:
+                pass
+            except Exception as error:
+                logger.error('Failed to open block %s.' % i, error=error)
+        return
+
+    def getPeerByHashId(self, hash):
+        '''
+            Return the pubkey of the user if known from the hash
+        '''
+        if self._core._crypto.pubKeyHashID() == hash:
+            retData = self._core._crypto.pubKey
+            return retData
+        conn = sqlite3.connect(self._core.peerDB)
+        c = conn.cursor()
+        command = (hash,)
+        retData = ''
+        for row in c.execute('SELECT ID FROM peers where hashID=?', command):
+            if row[0] != '':
+                retData = row[0]
+        return retData
+
+    def isCommunicatorRunning(self, timeout = 5, interval = 0.1):
+        try:
+            runcheck_file = 'data/.runcheck'
+
+            if os.path.isfile(runcheck_file):
+                os.remove(runcheck_file)
+                logger.debug('%s file appears to have existed before the run check.' % runcheck_file, timestamp = False)
+
+            self._core.daemonQueueAdd('runCheck')
+            starttime = time.time()
+
+            while True:
+                time.sleep(interval)
+                if os.path.isfile(runcheck_file):
+                    os.remove(runcheck_file)
+
+                    return True
+                elif time.time() - starttime >= timeout:
+                    return False
+        except:
+            return False
+
+    def token(self, size = 32):
+        '''
+            Generates a secure random hex encoded token
+        '''
+        return binascii.hexlify(os.urandom(size))
+
+    def importNewBlocks(self, scanDir=''):
+        '''
+            This function is intended to scan for new blocks ON THE DISK and import them
+        '''
+        blockList = self._core.getBlockList()
+        if scanDir == '':
+            scanDir = self._core.blockDataLocation
+        if not scanDir.endswith('/'):
+            scanDir += '/'
+        for block in glob.glob(scanDir + "*.dat"):
+            if block.replace(scanDir, '').replace('.dat', '') not in blockList:
+                logger.info('Found new block on dist %s' % block)
+                with open(block, 'rb') as newBlock:
+                    block = block.replace(scanDir, '').replace('.dat', '')
+                    if self._core._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''):
+                        self._core.addToBlockDB(block.replace('.dat', ''), dataSaved=True)
+                        logger.info('Imported block %s.' % block)
+                    else:
+                        logger.warn('Failed to verify hash for %s' % block)
+
+    def progressBar(self, value = 0, endvalue = 100, width = None):
+        '''
+            Outputs a progress bar with a percentage. Write \n after use.
+        '''
+
+        if width is None or height is None:
+            width, height = shutil.get_terminal_size((80, 24))
+
+        bar_length = width - 6
+
+        percent = float(value) / endvalue
+        arrow = '─' * int(round(percent * bar_length)-1) + '>'
+        spaces = ' ' * (bar_length - len(arrow))
+
+        sys.stdout.write("\r┣{0}┫ {1}%".format(arrow + spaces, int(round(percent * 100))))
+        sys.stdout.flush()
+    
+    def getEpoch(self):
+        '''returns epoch'''
+        return math.floor(time.time())
+
+def size(path='.'):
+    '''
+        Returns the size of a folder's contents in bytes
+    '''
+    total = 0
+    if os.path.exists(path):
+        if os.path.isfile(path):
+            total = os.path.getsize(path)
+        else:
+            for entry in os.scandir(path):
+                if entry.is_file():
+                    total += entry.stat().st_size
+                elif entry.is_dir():
+                    total += size(entry.path)
+    return total
+
+def humanSize(num, suffix='B'):
+    '''
+        Converts from bytes to a human readable format.
+    '''
+    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
+        if abs(num) < 1024.0:
+            return "%.1f %s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f %s%s" % (num, 'Yi', suffix)

onionr/static-data/bootstrap-nodes.txt

@@ -0,0 +1,2 @@
+onionragxuddecmg.onion
+dgyllprmtmym4gbk.onion

onionr/static-data/default-plugins/gui/info.json

@@ -0,0 +1,5 @@
+{
+    "name" : "gui",
+    "version" : "1.0",
+    "author" : "onionr"
+}

onionr/static-data/default-plugins/gui/main.py

@@ -0,0 +1,135 @@
+#!/usr/bin/python
+'''
+    Onionr - P2P Microblogging Platform & Social network
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+# Imports some useful libraries
+import logger, config, core
+import os, sqlite3, threading
+from onionrblockapi import Block
+
+plugin_name = 'gui'
+
+def send():
+    global message
+    block = Block()
+    block.setType('txt')
+    block.setContent(message)
+    logger.debug('Sent message in block %s.' % block.save(sign = True))
+
+
+def sendMessage():
+    global sendEntry
+
+    global message
+    message = sendEntry.get()
+
+    t = threading.Thread(target = send)
+    t.start()
+
+    sendEntry.delete(0, len(message))
+
+def update():
+    global listedBlocks, listbox, runningCheckDelayCount, runningCheckDelay, root, daemonStatus
+
+    for i in Block.getBlocks(type = 'txt'):
+        if i.getContent().strip() == '' or i.getHash() in listedBlocks:
+            continue
+        listbox.insert(99999, str(i.getContent()))
+        listedBlocks.append(i.getHash())
+        listbox.see(99999)
+
+    runningCheckDelayCount += 1
+
+    if runningCheckDelayCount == runningCheckDelay:
+        resp = pluginapi.daemon.local_command('ping')
+        if resp == 'pong':
+            daemonStatus.config(text = "Onionr Daemon Status: Running")
+        else:
+            daemonStatus.config(text = "Onionr Daemon Status: Not Running")
+        runningCheckDelayCount = 0
+    root.after(10000, update)
+
+
+def reallyOpenGUI():
+    import tkinter
+    global root, runningCheckDelay, runningCheckDelayCount, scrollbar, listedBlocks, nodeInfo, keyInfo, idText, idEntry, pubKeyEntry, listbox, daemonStatus, sendEntry
+
+    root = tkinter.Tk()
+
+    root.title("Onionr GUI")
+
+    runningCheckDelay = 5
+    runningCheckDelayCount = 4
+
+    scrollbar = tkinter.Scrollbar(root)
+    scrollbar.pack(side=tkinter.RIGHT, fill=tkinter.Y)
+
+    listedBlocks = []
+
+    nodeInfo = tkinter.Frame(root)
+    keyInfo = tkinter.Frame(root)
+
+    hostname = pluginapi.get_onionr().get_hostname()
+    logger.debug('Onionr Hostname: %s' % hostname)
+    idText = hostname
+
+    idEntry = tkinter.Entry(nodeInfo)
+    tkinter.Label(nodeInfo, text = "Node Address: ").pack(side=tkinter.LEFT)
+    idEntry.pack()
+    idEntry.insert(0, idText.strip())
+    idEntry.configure(state="readonly")
+
+    nodeInfo.pack()
+
+    pubKeyEntry = tkinter.Entry(keyInfo)
+
+    tkinter.Label(keyInfo, text="Public key: ").pack(side=tkinter.LEFT)
+
+    pubKeyEntry.pack()
+    pubKeyEntry.insert(0, pluginapi.get_core()._crypto.pubKey)
+    pubKeyEntry.configure(state="readonly")
+
+    keyInfo.pack()
+
+    sendEntry = tkinter.Entry(root)
+    sendBtn = tkinter.Button(root, text='Send Message', command=sendMessage)
+    sendEntry.pack(side=tkinter.TOP, pady=5)
+    sendBtn.pack(side=tkinter.TOP)
+
+    listbox = tkinter.Listbox(root, yscrollcommand=tkinter.Scrollbar.set, height=15)
+
+    listbox.pack(fill=tkinter.BOTH, pady=25)
+
+    daemonStatus = tkinter.Label(root, text="Onionr Daemon Status: unknown")
+    daemonStatus.pack()
+
+    scrollbar.config(command=tkinter.Listbox.yview)
+    root.after(2000, update)
+    root.mainloop()
+
+def openGUI():
+    t = threading.Thread(target = reallyOpenGUI)
+    t.daemon = False
+    t.start()
+
+def on_init(api, data = None):
+    global pluginapi
+    pluginapi = api
+
+    api.commands.register(['gui', 'launch-gui', 'open-gui'], openGUI)
+    api.commands.register_help('gui', 'Opens a graphical interface for Onionr')
+
+    return

onionr/static-data/default-plugins/pluginmanager/info.json

@@ -0,0 +1,5 @@
+{
+    "name" : "pluginmanager",
+    "version" : "1.0",
+    "author" : "onionr"
+}

onionr/static-data/default-plugins/pluginmanager/main.py

@@ -0,0 +1,546 @@
+'''
+    Onionr - P2P Microblogging Platform & Social network.
+
+    This plugin acts as a plugin manager, and allows the user to install other plugins distributed over Onionr.
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+# useful libraries
+import logger, config
+import os, sys, json, time, random, shutil, base64, getpass, datetime, re
+from onionrblockapi import Block
+
+plugin_name = 'pluginmanager'
+
+keys_data = {'keys' : {}, 'plugins' : [], 'repositories' : {}}
+
+# key functions
+
+def writeKeys():
+    '''
+        Serializes and writes the keystore in memory to file
+    '''
+
+    file = open(keys_file, 'w')
+    file.write(json.dumps(keys_data, indent=4, sort_keys=True))
+    file.close()
+
+def readKeys():
+    '''
+        Loads the keystore into memory
+    '''
+
+    global keys_data
+    keys_data = json.loads(open(keys_file).read())
+    return keys_data
+
+def getKey(plugin):
+    '''
+        Returns the public key for a given plugin
+    '''
+
+    global keys_data
+    readKeys()
+    return (keys_data['keys'][plugin] if plugin in keys_data['keys'] else None)
+
+def saveKey(plugin, key):
+    '''
+        Saves the public key for a plugin to keystore
+    '''
+
+    global keys_data
+    readKeys()
+    keys_data['keys'][plugin] = key
+    writeKeys()
+
+def getPlugins():
+    '''
+        Returns a list of plugins installed by the plugin manager
+    '''
+
+    global keys_data
+    readKeys()
+    return keys_data['plugins']
+
+def addPlugin(plugin):
+    '''
+        Saves the plugin name, to remember that it was installed by the pluginmanager
+    '''
+
+    global keys_data
+    readKeys()
+    if not plugin in keys_data['plugins']:
+        keys_data['plugins'].append(plugin)
+        writeKeys()
+
+def removePlugin(plugin):
+    '''
+        Removes the plugin name from the pluginmanager's records
+    '''
+
+    global keys_data
+    readKeys()
+    if plugin in keys_data['plugins']:
+        keys_data['plugins'].remove(plugin)
+        writeKeys()
+
+def getRepositories():
+    '''
+        Returns a list of plugins installed by the plugin manager
+    '''
+
+    global keys_data
+    readKeys()
+    return keys_data['repositories']
+
+def addRepository(repositories, data):
+    '''
+        Saves the plugin name, to remember that it was installed by the pluginmanager
+    '''
+
+    global keys_data
+    readKeys()
+    keys_data['repositories'][repositories] = data
+    writeKeys()
+
+def removeRepository(repositories):
+    '''
+        Removes the plugin name from the pluginmanager's records
+    '''
+
+    global keys_data
+    readKeys()
+    if plugin in keys_data['repositories']:
+        del keys_data['repositories'][repositories]
+        writeKeys()
+
+def check():
+    '''
+        Checks to make sure the keystore file still exists
+    '''
+
+    global keys_file
+    keys_file = pluginapi.plugins.get_data_folder(plugin_name) + 'keystore.json'
+    if not os.path.isfile(keys_file):
+        writeKeys()
+
+# plugin management
+
+def sanitize(name):
+    return re.sub('[^0-9a-zA-Z]+', '', str(name).lower())[:255]
+
+def blockToPlugin(block):
+    try:
+        block = Block(block)
+        blockContent = json.loads(block.getContent())
+
+        name = sanitize(blockContent['name'])
+        author = blockContent['author']
+        date = blockContent['date']
+        version = None
+
+        if 'version' in blockContent['info']:
+            version = blockContent['info']['version']
+
+        content = base64.b64decode(blockContent['content'].encode())
+
+        source = pluginapi.plugins.get_data_folder(plugin_name) + 'plugin.zip'
+        destination = pluginapi.plugins.get_folder(name)
+
+        with open(source, 'wb') as f:
+            f.write(content)
+
+        if os.path.exists(destination) and not os.path.isfile(destination):
+            shutil.rmtree(destination)
+
+        shutil.unpack_archive(source, destination)
+        pluginapi.plugins.enable(name)
+
+        logger.info('Installation of %s complete.' % name)
+
+        return True
+    except Exception as e:
+        logger.error('Failed to install plugin.', error = e, timestamp = False)
+
+    return False
+
+def pluginToBlock(plugin, import_block = True):
+    try:
+        plugin = sanitize(plugin)
+
+        directory = pluginapi.get_pluginapi().get_folder(plugin)
+        data_directory = pluginapi.get_pluginapi().get_data_folder(plugin)
+        zipfile = pluginapi.get_pluginapi().get_data_folder(plugin_name) + 'plugin.zip'
+
+        if os.path.exists(directory) and not os.path.isfile(directory):
+            if os.path.exists(data_directory) and not os.path.isfile(data_directory):
+                shutil.rmtree(data_directory)
+            if os.path.exists(zipfile) and os.path.isfile(zipfile):
+                os.remove(zipfile)
+            if os.path.exists(directory + '__pycache__') and not os.path.isfile(directory + '__pycache__'):
+                shutil.rmtree(directory + '__pycache__')
+
+            shutil.make_archive(zipfile[:-4], 'zip', directory)
+            data = base64.b64encode(open(zipfile, 'rb').read())
+
+            author = getpass.getuser()
+            description = 'Default plugin description'
+            info = {"name" : plugin}
+            try:
+                if os.path.exists(directory + 'info.json'):
+                    info = json.loads(open(directory + 'info.json').read())
+                    if 'author' in info:
+                        author = info['author']
+                    if 'description' in info:
+                        description = info['description']
+            except:
+                pass
+
+            metadata = {'author' : author, 'date' : str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')), 'name' : plugin, 'info' : info, 'compiled-by' : plugin_name, 'content' : data.decode('utf-8'), 'description' : description}
+
+            hash = pluginapi.get_core().insertBlock(json.dumps(metadata), header = 'plugin', sign = True)
+
+            if import_block:
+                pluginapi.get_utils().importNewBlocks()
+
+            return hash
+        else:
+            logger.error('Plugin %s does not exist.' % plugin)
+    except Exception as e:
+        logger.error('Failed to convert plugin to block.', error = e, timestamp = False)
+
+    return False
+
+def installBlock(block):
+    try:
+        block = Block(block)
+        blockContent = json.loads(block.getContent())
+
+        name = sanitize(blockContent['name'])
+        author = blockContent['author']
+        date = blockContent['date']
+        version = None
+
+        if 'version' in blockContent['info']:
+            version = blockContent['info']['version']
+
+        install = False
+
+        logger.info(('Will install %s' + (' v' + version if not version is None else '') + ' (%s), by %s') % (name, date, author))
+
+        # TODO: Convert to single line if statement
+        if os.path.exists(pluginapi.plugins.get_folder(name)):
+            install = logger.confirm(message = 'Continue with installation (will overwrite existing plugin) %s?')
+        else:
+            install = logger.confirm(message = 'Continue with installation %s?')
+
+        if install:
+            blockToPlugin(block.getHash())
+            addPlugin(name)
+        else:
+            logger.info('Installation cancelled.')
+            return False
+
+        return True
+    except Exception as e:
+        logger.error('Failed to install plugin.', error = e, timestamp = False)
+        return False
+
+def uninstallPlugin(plugin):
+    try:
+        plugin = sanitize(plugin)
+
+        pluginFolder = pluginapi.plugins.get_folder(plugin)
+        exists = (os.path.exists(pluginFolder) and not os.path.isfile(pluginFolder))
+        installedByPluginManager = plugin in getPlugins()
+        remove = False
+
+        if not exists:
+            logger.warn('Plugin %s does not exist.' % plugin, timestamp = False)
+            return False
+
+        default = 'y'
+        if not installedByPluginManager:
+            logger.warn('The plugin %s was not installed by %s.' % (plugin, plugin_name), timestamp = False)
+            default = 'n'
+        remove = logger.confirm(message = 'All plugin data will be lost. Are you sure you want to proceed %s?', default = default)
+
+        if remove:
+            if installedByPluginManager:
+                removePlugin(plugin)
+            pluginapi.plugins.disable(plugin)
+            shutil.rmtree(pluginFolder)
+
+            logger.info('Uninstallation of %s complete.' % plugin)
+
+            return True
+        else:
+            logger.info('Uninstallation cancelled.')
+    except Exception as e:
+        logger.error('Failed to uninstall plugin.', error = e)
+    return False
+
+# command handlers
+
+def help():
+    logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]')
+    logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]')
+
+def commandInstallPlugin():
+    if len(sys.argv) >= 3:
+        check()
+
+        pluginname = sys.argv[2]
+        pkobh = None # public key or block hash
+
+        version = None
+        if ':' in pluginname:
+            details = pluginname
+            pluginname = sanitize(details[0])
+            version = details[1]
+
+        sanitize(pluginname)
+
+        if len(sys.argv) >= 4:
+            # public key or block hash specified
+            pkobh = sys.argv[3]
+        else:
+            # none specified, check if in config file
+            pkobh = getKey(pluginname)
+
+        if pkobh is None:
+            # still nothing found, try searching repositories
+            logger.info('Searching for public key in repositories...')
+            try:
+                repos = getRepositories()
+                distributors = list()
+                for repo, records in repos.items():
+                    if pluginname in records:
+                        logger.debug('Found %s in repository %s for plugin %s.' % (records[pluginname], repo, pluginname))
+                        distributors.append(records[pluginname])
+
+                if len(distributors) != 0:
+                    distributor = None
+
+                    if len(distributors) == 1:
+                        logger.info('Found distributor: %s' % distributors[0])
+                        distributor = distributors[0]
+                    else:
+                        distributors_message = ''
+
+                        index = 1
+                        for dist in distributors:
+                            distributors_message += '    ' + logger.colors.bold + str(index) + ') ' + logger.colors.reset + str(dist) + '\n'
+                            index += 1
+
+                        logger.info((logger.colors.bold + 'Found distributors (%s):' + logger.colors.reset + '\n' + distributors_message) % len(distributors))
+
+                        valid = False
+                        while not valid:
+                            choice = logger.readline('Select the number of the key to use, from 1 to %s, or press Ctrl+C to cancel:' % (index - 1))
+
+                            try:
+                                if int(choice) < index and int(choice) >= 1:
+                                    distributor = distributors[int(choice)]
+                                    valid = True
+                            except KeyboardInterrupt:
+                                logger.info('Installation cancelled.')
+                                return True
+                            except:
+                                pass
+
+                    if not distributor is None:
+                        pkobh = distributor
+            except Exception as e:
+                logger.warn('Failed to lookup plugin in repositories.', timestamp = False)
+                logger.error('asdf', error = e, timestamp = False)
+
+        if pkobh is None:
+            logger.error('No key for this plugin found in keystore or repositories, please specify.')
+            help()
+            return True
+
+        valid_hash = pluginapi.get_utils().validateHash(pkobh)
+        real_block = False
+        valid_key = pluginapi.get_utils().validatePubKey(pkobh)
+        real_key = False
+
+        if valid_hash:
+            real_block = Block.exists(pkobh)
+        elif valid_key:
+            real_key = pluginapi.get_utils().hasKey(pkobh)
+
+        blockhash = None
+
+        if valid_hash and not real_block:
+            logger.error('Block hash not found. Perhaps it has not been synced yet?')
+            logger.debug('Is valid hash, but does not belong to a known block.')
+
+            return True
+        elif valid_hash and real_block:
+            blockhash = str(pkobh)
+            logger.debug('Using block %s...' % blockhash)
+
+            installBlock(blockhash)
+        elif valid_key and not real_key:
+            logger.error('Public key not found. Try adding the node by address manually, if possible.')
+            logger.debug('Is valid key, but the key is not a known one.')
+        elif valid_key and real_key:
+            publickey = str(pkobh)
+            logger.debug('Using public key %s...' % publickey)
+
+            saveKey(pluginname, pkobh)
+
+            signedBlocks = Block.getBlocks(type = 'plugin', signed = True, signer = publickey)
+
+            mostRecentTimestamp = None
+            mostRecentVersionBlock = None
+
+            for block in signedBlocks:
+                try:
+                    blockContent = json.loads(block.getContent())
+
+                    if not (('author' in blockContent) and ('info' in blockContent) and ('date' in blockContent) and ('name' in blockContent)):
+                        raise ValueError('Missing required parameter `date` in block %s.' % block.getHash())
+
+                    blockDatetime = datetime.datetime.strptime(blockContent['date'], '%Y-%m-%d %H:%M:%S')
+
+                    if blockContent['name'] == pluginname:
+                        if ('version' in blockContent['info']) and (blockContent['info']['version'] == version) and (not version is None):
+                            mostRecentTimestamp = blockDatetime
+                            mostRecentVersionBlock = block.getHash()
+                            break
+                        elif mostRecentTimestamp is None:
+                            mostRecentTimestamp = blockDatetime
+                            mostRecentVersionBlock = block.getHash()
+                        elif blockDatetime > mostRecentTimestamp:
+                            mostRecentTimestamp = blockDatetime
+                            mostRecentVersionBlock = block.getHash()
+                except Exception as e:
+                    pass
+
+            logger.warn('Only continue the installation is you are absolutely certain that you trust the plugin distributor. Public key of plugin distributor: %s' % publickey, timestamp = False)
+            installBlock(mostRecentVersionBlock)
+        else:
+            logger.error('Unknown data "%s"; must be public key or block hash.' % str(pkobh))
+            return
+    else:
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin> [public key/block hash]')
+
+    return True
+
+def commandUninstallPlugin():
+    if len(sys.argv) >= 3:
+        uninstallPlugin(sys.argv[2])
+    else:
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin>')
+
+    return True
+
+def commandSearchPlugin():
+    logger.info('This feature has not been created yet. Please check back later.')
+    return True
+
+def commandAddRepository():
+    if len(sys.argv) >= 3:
+        check()
+
+        blockhash = sys.argv[2]
+
+        if pluginapi.get_utils().validateHash(blockhash):
+            if Block.exists(blockhash):
+                try:
+                    blockContent = json.loads(Block(blockhash).getContent())
+
+                    pluginslist = dict()
+
+                    for pluginname, distributor in blockContent['plugins'].items():
+                        if pluginapi.get_utils().validatePubKey(distributor):
+                            pluginslist[pluginname] = distributor
+
+                    logger.debug('Found %s records in repository.' % len(pluginslist))
+
+                    if len(pluginslist) != 0:
+                        addRepository(blockhash, pluginslist)
+                        logger.info('Successfully added repository.')
+                    else:
+                        logger.error('Repository contains no records, not importing.')
+                except Exception as e:
+                    logger.error('Failed to parse block.', error = e)
+            else:
+                logger.error('Block hash not found. Perhaps it has not been synced yet?')
+                logger.debug('Is valid hash, but does not belong to a known block.')
+        else:
+            logger.error('Unknown data "%s"; must be block hash.' % str(pkobh))
+    else:
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [block hash]')
+
+    return True
+
+def commandRemoveRepository():
+    if len(sys.argv) >= 3:
+        check()
+
+        blockhash = sys.argv[2]
+
+        if pluginapi.get_utils().validateHash(blockhash):
+            if blockhash in getRepositories():
+                try:
+                    removeRepository(blockhash)
+                except Exception as e:
+                    logger.error('Failed to parse block.', error = e)
+            else:
+                logger.error('Repository has not been imported, nothing to remove.')
+        else:
+            logger.error('Unknown data "%s"; must be block hash.' % str(pkobh))
+    else:
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' [block hash]')
+
+    return True
+
+def commandPublishPlugin():
+    if len(sys.argv) >= 3:
+        check()
+
+        pluginname = sanitize(sys.argv[2])
+        pluginfolder = pluginapi.plugins.get_folder(pluginname)
+
+        if os.path.exists(pluginfolder) and not os.path.isfile(pluginfolder):
+            block = pluginToBlock(pluginname)
+            logger.info('Plugin saved in block %s.' % block)
+        else:
+            logger.error('Plugin %s does not exist.' % pluginname, timestamp = False)
+    else:
+        logger.info(sys.argv[0] + ' ' + sys.argv[1] + ' <plugin>')
+
+# event listeners
+
+def on_init(api, data = None):
+    global pluginapi
+    pluginapi = api
+    check()
+
+    # register some commands
+    api.commands.register(['install-plugin', 'installplugin', 'plugin-install', 'install', 'plugininstall'], commandInstallPlugin)
+    api.commands.register(['remove-plugin', 'removeplugin', 'plugin-remove', 'uninstall-plugin', 'uninstallplugin', 'plugin-uninstall', 'uninstall', 'remove', 'pluginremove'], commandUninstallPlugin)
+    api.commands.register(['search', 'filter-plugins', 'search-plugins', 'searchplugins', 'search-plugin', 'searchplugin', 'findplugin', 'find-plugin', 'filterplugin', 'plugin-search', 'pluginsearch'], commandSearchPlugin)
+    api.commands.register(['add-repo', 'add-repository', 'addrepo', 'addrepository', 'repository-add', 'repo-add', 'repoadd', 'addrepository', 'add-plugin-repository', 'add-plugin-repo', 'add-pluginrepo', 'add-pluginrepository', 'addpluginrepo', 'addpluginrepository'], commandAddRepository)
+    api.commands.register(['remove-repo', 'remove-repository', 'removerepo', 'removerepository', 'repository-remove', 'repo-remove', 'reporemove', 'removerepository', 'remove-plugin-repository', 'remove-plugin-repo', 'remove-pluginrepo', 'remove-pluginrepository', 'removepluginrepo', 'removepluginrepository', 'rm-repo', 'rm-repository', 'rmrepo', 'rmrepository', 'repository-rm', 'repo-rm', 'reporm', 'rmrepository', 'rm-plugin-repository', 'rm-plugin-repo', 'rm-pluginrepo', 'rm-pluginrepository', 'rmpluginrepo', 'rmpluginrepository'], commandRemoveRepository)
+    api.commands.register(['publish-plugin', 'plugin-publish', 'publishplugin', 'pluginpublish', 'publish'], commandPublishPlugin)
+
+    # add help menus once the features are actually implemented
+
+    return

onionr/static-data/default_config.json

@@ -0,0 +1,20 @@
+{
+    "devmode": true,
+    "dc_response": true,
+
+    "log": {
+        "file": {
+            "output": true,
+            "path": "data/output.log"
+        },
+
+        "console": {
+            "output": true,
+            "color": true
+        }
+    },
+    "allocations":{
+        "disk": 1000000000,
+        "netTotal": 1000000000
+    }
+}

onionr/static-data/default_plugin.py

@@ -0,0 +1,43 @@
+'''
+    $name plugin template file.
+    Generated on $date by $user.
+'''
+
+# Imports some useful libraries
+import logger, config
+
+plugin_name = '$name'
+
+def on_init(api, data = None):
+    '''
+        This event is called after Onionr is initialized, but before the command
+        inputted is executed. Could be called when daemon is starting or when
+        just the client is running.
+    '''
+
+    # Doing this makes it so that the other functions can access the api object
+    # by simply referencing the variable `pluginapi`.
+    global pluginapi
+    pluginapi = api
+
+    return
+
+def on_start(api, data = None):
+    '''
+        This event can be called for multiple reasons:
+        1) The daemon is starting
+        2) The user called `onionr --start-plugins` or `onionr --reload-plugins`
+        3) For whatever reason, the plugins are reloading
+    '''
+
+    return
+
+def on_stop(api, data = None):
+    '''
+        This event can be called for multiple reasons:
+        1) The daemon is stopping
+        2) The user called `onionr --stop-plugins` or `onionr --reload-plugins`
+        3) For whatever reason, the plugins are reloading
+    '''
+
+    return

onionr/static-data/index.html

@@ -0,0 +1,5 @@
+<h1>This is an Onionr Node</h1>
+
+<p>The content on this server is not necessarily created or intentionally stored by the owner of the software.</p>
+
+<p>To learn more about Onionr, see the website at <a href="https://onionr.voidnet.tech/">https://Onionr.VoidNet.tech/</a></p>

onionr/tests.py

@@ -14,7 +14,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import unittest, sys, os, base64, tarfile, shutil, simplecrypt, logger, btc
+import unittest, sys, os, base64, tarfile, shutil, simplecrypt, logger #, btc
 
 class OnionrTests(unittest.TestCase):
     def testPython3(self):
@@ -56,7 +56,7 @@ class OnionrTests(unittest.TestCase):
         myCore = core.Core()
         if not os.path.exists('data/peers.db'):
             myCore.createPeerDB()
-        if myCore.addPeer('6M5MXL237OK57ITHVYN5WGHANPGOMKS5C3PJLHBBNKFFJQOIDOJA====') and not myCore.addPeer('NFXHMYLMNFSAU==='):
+        if myCore.addPeer('6M5MXL237OK57ITHVYN5WGHANPGOMKS5C3PJLHBBNKFFJQOIDOJA====', '1cSix9Ao/yQSdo0sNif8cm2uTcYnSphb4JdZL/3WkN4=') and not myCore.addPeer('NFXHMYLMNFSAU===', '1cSix9Ao/yQSdo0sNif8cm2uTcYnSphb4JdZL/3WkN4='):
             self.assertTrue(True)
         else:
             self.assertTrue(False)
@@ -129,7 +129,14 @@ class OnionrTests(unittest.TestCase):
         logger.debug('-'*26 + '\n')
         logger.info('Running simple plugin reload test...')
 
-        import onionrplugins
+        import onionrplugins, os
+
+        if not onionrplugins.exists('test'):
+            os.makedirs(onionrplugins.get_plugins_folder('test'))
+            with open(onionrplugins.get_plugins_folder('test') + '/main.py', 'a') as main:
+                main.write("print('Running')\n\ndef on_test(pluginapi, data = None):\n    print('received test event!')\n    return True\n\ndef on_start(pluginapi, data = None):\n    print('start event called')\n\ndef on_stop(pluginapi, data = None):\n    print('stop event called')\n\ndef on_enable(pluginapi, data = None):\n    print('enable event called')\n\ndef on_disable(pluginapi, data = None):\n    print('disable event called')\n")
+            onionrplugins.enable('test')
+
         try:
             onionrplugins.reload('test')
             self.assertTrue(True)
@@ -140,7 +147,14 @@ class OnionrTests(unittest.TestCase):
         logger.debug('-'*26 + '\n')
         logger.info('Running simple plugin restart test...')
 
-        import onionrplugins
+        import onionrplugins, os
+
+        if not onionrplugins.exists('test'):
+            os.makedirs(onionrplugins.get_plugins_folder('test'))
+            with open(onionrplugins.get_plugins_folder('test') + '/main.py', 'a') as main:
+                main.write("print('Running')\n\ndef on_test(pluginapi, data = None):\n    print('received test event!')\n    return True\n\ndef on_start(pluginapi, data = None):\n    print('start event called')\n\ndef on_stop(pluginapi, data = None):\n    print('stop event called')\n\ndef on_enable(pluginapi, data = None):\n    print('enable event called')\n\ndef on_disable(pluginapi, data = None):\n    print('disable event called')\n")
+            onionrplugins.enable('test')
+
         try:
             onionrplugins.start('test')
             onionrplugins.stop('test')
@@ -152,13 +166,24 @@ class OnionrTests(unittest.TestCase):
         logger.debug('-'*26 + '\n')
         logger.info('Running plugin event test...')
 
-        import onionrplugins as plugins, onionrevents as events
+        import onionrplugins as plugins, onionrevents as events, os
+
+        if not plugins.exists('test'):
+            os.makedirs(plugins.get_plugins_folder('test'))
+            with open(plugins.get_plugins_folder('test') + '/main.py', 'a') as main:
+                main.write("print('Running')\n\ndef on_test(pluginapi, data = None):\n    print('received test event!')\n    print('thread test started...')\n    import time\n    time.sleep(1)\n    \n    return True\n\ndef on_start(pluginapi, data = None):\n    print('start event called')\n\ndef on_stop(pluginapi, data = None):\n    print('stop event called')\n\ndef on_enable(pluginapi, data = None):\n    print('enable event called')\n\ndef on_disable(pluginapi, data = None):\n    print('disable event called')\n")
+            plugins.enable('test')
+
 
         plugins.start('test')
-        if not events.call(plugins.get_plugin('test'), 'test'):
+        if not events.call(plugins.get_plugin('test'), 'enable'):
             self.assertTrue(False)
 
-        events.event('test', data = {'tests': self})
+        logger.debug('preparing to start thread', timestamp = False)
+        thread = events.event('test', data = {'tests': self})
+        logger.debug('thread running...', timestamp = False)
+        thread.join()
+        logger.debug('thread finished.', timestamp = False)
 
         self.assertTrue(True)
 

onionr/timedhmac.py

@@ -12,7 +12,9 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
+
 import hmac, base64, time, math
+
 class TimedHMAC:
     def __init__(self, base64Key, data, hashAlgo):
         '''
@@ -23,6 +25,7 @@ class TimedHMAC:
 
             Maximum of 10 seconds grace period
         '''
+        
         self.data = data
         self.expire = math.floor(time.time())
         self.hashAlgo = hashAlgo
@@ -34,11 +37,14 @@ class TimedHMAC:
         return
 
     def check(self, data):
-        # Check a hash (and verify time is sane)
+        '''
+            Check a hash (and verify time is sane)
+        '''
+        
         testHash = hmac.HMAC(base64.b64decode(base64Key).decode(), digestmod=self.hashAlgo)
         testHash.update(data + math.floor(time.time()))
         testHash = testHash.hexdigest()
         if hmac.compare_digest(testHash, self.HMACResult):
             return true
-        else:
+        
         return false

requirements.txt

@@ -1,8 +1,11 @@
-PyNaCl==1.2.1
-requests==2.12.4
-Flask==0.12.2
-simple_crypt==4.1.7
 urllib3==1.19.1
+gevent==1.2.2
+PyNaCl==1.2.1
+pycoin==0.62
+Flask==1.0
 sha3==0.2.1
-PySocks==1.6.8
+simple_crypt==4.1.7
-bitpeer.py==0.4.7.5
+ecdsa==0.13
+requests==2.12.4
+SocksiPy_branch==1.01
+sphinx_rtd_theme==0.3.0