finish merging new-main

.gitignore

@@ -3,6 +3,7 @@ onionr/data/config.ini
 onionr/data/*.db
 onionr/data-old/*
 onionr/data*
+onionr/testdata
 onionr/*.pyc
 onionr/*.log
 onionr/data/hs/hostname

LICENSE.txt

@@ -1,5 +1,4 @@
-Onionr Logo is licensed under Creative Commons Attribution-Share Alike 3.0 Unported
+The Onionr logo was created by [Anhar Ismail](https://github.com/anharismail) under the [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/). 
-https://creativecommons.org/licenses/by-sa/4.0/
 
 
                     GNU GENERAL PUBLIC LICENSE

Makefile

@@ -1,3 +1,6 @@
+ONIONR_HOME ?= data
+all:;: '$(ONIONR_HOME)'
+
 PREFIX = /usr/local
 
 .DEFAULT_GOAL := setup

README.md

@@ -31,43 +31,79 @@ Onionr can be used for mail, as a social network, instant messenger, file sharin
 
 ## Main Features
 
-* [X] Fully p2p/decentralized, no trackers or other single points of failure
+* [X] 🌐 Fully p2p/decentralized, no trackers or other single points of failure
-* [X] End to end encryption of user data
+* [X] 🔒 End to end encryption of user data
-* [X] Optional non-encrypted blocks, useful for blog posts or public file sharing
+* [X] 📢 Optional non-encrypted blocks, useful for blog posts or public file sharing
-* [X] Easy API system for integration to websites
+* [X] 💻 Easy API system for integration to websites
-* [X] Metadata analysis resistance
+* [X] 🕵️ Metadata analysis resistance and anonymity
-* [X] Transport agnosticism (no internet required)
+* [X] 📡 Transport agnosticism (no internet required)
 
 **Onionr API and functionality is subject to non-backwards compatible change during pre-alpha development**
 
+# Screenshots
+
+<img alt='Node statistics page screenshot' src='docs/onionr-1.png' width=600>
+
+Node statistics
+
+<img alt='Friend/contact manager screenshot' src='docs/onionr-2.png' width=600>
+
+Friend/contact manager
+
+<img alt='Encrypted, metadata-masking mail application screenshot' src='docs/onionr-3.png' width=600>
+
+Encrypted, metadata-masking mail application.
+
 # Install and Run on Linux
 
 The following applies to Ubuntu Bionic. Other distros may have different package or command names.
 
-* Have python3.5+, python3-pip, Tor (daemon, not browser) installed (python3-dev recommended)
+* Have python3.6+, python3-pip, Tor (daemon, not browser) installed (python3-dev recommended)
 * Clone the git repo: `$ git clone https://gitlab.com/beardog/onionr`
 * cd into install direction: `$ cd onionr/`
-* Install the Python dependencies ([virtualenv strongly recommended](https://virtualenv.pypa.io/en/stable/userguide/)): `$ pip3 install -r requirements.txt`
+* Install the Python dependencies ([virtualenv strongly recommended](https://virtualenv.pypa.io/en/stable/userguide/)): `$ pip3 install --require-hashes -r requirements.txt`
+
+(--require-hashes is intended to prevent exploitation via compromise of Pypi/CA certificates)
 
 ## Help out
 
 Everyone is welcome to help out. Help is wanted for the following:
 
 * Development (Get in touch first)
-    * Creation of a lib for use from other languages and faster proof-of-work
+    * Creation of a shared lib for use from other languages and faster proof-of-work
     * Android and IOS development
-    * Windows and Mac support
+    * Windows and Mac support (already partially supported, testers needed)
     * General bug fixes and development of new features
 * Testing
+* UI/UX design
 * Running stable nodes
 * Security review/audit
 * Automatic I2P setup
 
-Bitcoin: [1onion55FXzm6h8KQw3zFw2igpHcV7LPq](bitcoin:1onion55FXzm6h8KQw3zFw2igpHcV7LPq)
+Contribute money:
-USD: [Ko-Fi](https://www.ko-fi.com/beardogkf)
+
+Donating at least $5 gets you cool Onionr stickers. Get in touch if you want them.
+
+Bitcoin: [1onion55FXzm6h8KQw3zFw2igpHcV7LPq](bitcoin:1onion55FXzm6h8KQw3zFw2igpHcV7LPq) (Contact us for privacy coins like Monero)
+
+USD (Card/Paypal): [Ko-Fi](https://www.ko-fi.com/beardogkf)
+
+Note: probably not tax deductible
+
+## Contact
+
+beardog [ at ] mailbox.org
 
 ## Disclaimer
 
-The Tor Project, I2P developers, and anyone else do not own, create, or endorse this project, and are not otherwise involved.
+The Tor Project and I2P developers do not own, create, or endorse this project, and are not otherwise involved.
+
+Tor is a trademark for the Tor Project. We do not own it.
 
 The 'open source badge' is by Maik Ellerbrock and is licensed under a Creative Commons Attribution 4.0 International License.
+
+## Logo
+
+The Onionr logo was created by [Anhar Ismail](https://github.com/anharismail) under the [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/). 
+
+If you modify and redistribute our code ("forking"), please use a different logo and project name to avoid confusion. Please do not use our logo in a way that makes it seem like we endorse you without permission.

docs/whitepaper.md

@@ -1,25 +1,25 @@
 <p align="center">
- <img src="onionr-logo.png" alt="<h1>Onionr</h1>">
+ <img src="onionr-logo.png" alt="<h1>Onionr</h1>" width=200>
 </p>
 <p align="center">Anonymous, Decentralized, Distributed Network</p>
 
 # Introduction
 
-One of the most important things in the modern world is information. The ability to communicate freely with others is crucial for maintaining personal liberties. The internet has provided humanity with the ability to spread information globally, but there are many people who try (and sometimes succeed) to stifle the flow of information.
+We believe that the ability to communicate freely with others is crucial for maintaining societal and personal liberty. The internet has provided humanity with the ability to spread information globally, but there are many persons and organizations who try to stifle the flow of information, sometimes with success.
 
-Internet censorship comes in many forms, state censorship, corporate consolidation of media, threats of violence, network exploitation (e.g. denial of service attacks).
+Internet censorship comes in many forms, state censorship, corporate consolidation of media, threats of violence, network exploitation (e.g. denial of service attacks) and other threats.
 
-To prevent censorship or loss of information, these measures must be in place:
+We hold that in order to protect individual privacy, users must have the ability to communicate anonymously and with decentralization. 
 
-* Resistance to censorship of underlying infrastructure or of network hosts
+We believe that in order to prevent censorship and loss of information, these measures must be in place:
+
+* Resistance to censorship of underlying infrastructure or of particular network hosts
 
 * Anonymization of users by default
-   * The Inability to coerce human users (personal threats/"doxxing", or totalitarian regime censorship)
+   * The Inability to coerce users (personal threats/"doxxing", or totalitarian regime censorship)
 
 * Economic availability. A system should not rely on a single device to be constantly online, and should not be overly expensive to use. The majority of people in the world own cell phones, but comparatively few own personal computers, particularly in developing countries. Internet connectivity can be slow or spotty in many areas.
 
-There are many great projects that tackle decentralization and privacy issues, but there are none which tackle all of the above issues. Some of the existing networks have also not worked well in practice, or are more complicated than they need to be.
-
 # Onionr Design Goals
 
 When designing Onionr we had these main goals in mind:
@@ -91,6 +91,54 @@ In addition, randomness beacons such as the one operated by [NIST](https://beaco
 
 # Direct Connections
 
-We propose a method of using Onionr's block sync system to enable direct connections between peers by having one peer request to connect to another using the peer's public key. Since the request is within a standard block, proof of work must be used to request connection. If the requested peer is available and wishes to accept the connection,Onionr will generate a temporary .onion address for the other peer to connect to. Alternatively, a reverse connection may be formed, which is faster to establish but requires a message brokering system instead of a standard socket.
+We propose a method of using Onionr's block sync system to enable direct connections between peers by having one peer request to connect to another using the peer's public key. Since the request is within a standard block, proof of work must be used to request connection. If the requested peer is available and wishes to accept the connection, Onionr will generate a temporary .onion address for the other peer to connect to. Alternatively, a reverse connection may be formed, which is faster to establish but requires a message brokering system instead of a standard socket.
 
 The benefits of such a system are increased privacy, and the ability to anonymously communicate from multiple devices at once. In a traditional onion service, one's online status can be monitored and more easily correlated.
+
+# Threat Model
+
+The goal of Onionr is to provide a method of distributing information in a manner in which the difficulty of discovering the identity of those sending and receiving the information is greatly increased. In this section we detail what information we want to protect and who we're protecting it from. 
+
+In this threat model, "protected" means available in plaintext only to those which it was intended, and regardless non-malleable
+
+## Threat Actors
+
+Onionr assumes that traffic/data is being surveilled by powerful actors on every level but the user's device.
+
+We also assume that the actors are capable of the following:
+
+* Running tens of thousands of Onionr nodes
+* Surveiling most of the Tor and I2P networks
+
+## Protected Data
+
+We seek to protect the following information:
+
+* Contents of private data. E.g. 'mail' messages and secret files
+* Relationship metadata. Unless something is desired to be published publicly, we seek to hide the creator and recipients of such data.
+* Physical location/IP address of nodes on the network
+* All block data from tampering
+
+### Data we cannot or do not protect
+
+* Data specifically inserted as plaintext is available to the public
+* The public key of signed plaintext blocks
+* The fact that one is using Tor or I2P
+   * The fact that one is using Onionr specifically can likely be discovered using long term traffic analysis
+   * Intense traffic analysis may be able to discover what node created a block. For this reason we offer a high security setting to only share blocks via uploads that we recommend for those who need the best privacy.
+
+## Assumptions
+
+We assume that Tor onion services (v3) and I2P services cannot be trivially deanonymized, and that the underlying cryptographic primitives we employ cannot be broken in any manner faster than brute force unless a quantum computer is used. 
+
+Once quantum safe algorithms are more mature and have decent high level libraries, they will be deployed.
+
+# Comparisons to other P2P software
+
+Since Onionr is far from the first to implement many of these ideas (on their own), this section compares Onionr to other networks, using points we consider to be the most important.
+
+![network comparison image](network-comparison.png)
+
+# Conclusion
+
+If successful, Onionr will be a complete decentralized platform for anonymous computing, complete with limited metadata exposure, both node and user anonymity, and spam prevention

onionr/api.py

@@ -21,10 +21,13 @@ from gevent.pywsgi import WSGIServer, WSGIHandler
 from gevent import Timeout
 import flask, cgi, uuid
 from flask import request, Response, abort, send_from_directory
-import sys, random, threading, hmac, hashlib, base64, time, math, os, json, socket
+import sys, random, threading, hmac, base64, time, os, json, socket
 import core
 from onionrblockapi import Block
-import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionr
+import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config
+import httpapi
+from httpapi import friendsapi, simplecache
+import onionr
 
 class FDSafeHandler(WSGIHandler):
     '''Our WSGI handler. Doesn't do much non-default except timeouts'''
@@ -38,9 +41,9 @@ class FDSafeHandler(WSGIHandler):
 
 def setBindIP(filePath):
     '''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)'''
+    if config.get('general.random_bind_ip', True):
         hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
         data = '.'.join(hostOctets)
-    
         # Try to bind IP. Some platforms like Mac block non normal 127.x.x.x
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         try:
@@ -50,10 +53,10 @@ def setBindIP(filePath):
             logger.warn('Your platform appears to not support random local host addresses 127.x.x.x. Falling back to 127.0.0.1.')
             data = '127.0.0.1'
         s.close()
-    
+    else:
+        data = '127.0.0.1'
     with open(filePath, 'w') as bindFile:
         bindFile.write(data)
-
     return data
 
 class PublicAPI:
@@ -173,7 +176,7 @@ class PublicAPI:
             try:
                 newNode = request.form['node'].encode()
             except KeyError:
-                logger.warn('No block specified for upload')
+                logger.warn('No node specified for upload')
                 pass
             else:
                 try:
@@ -230,7 +233,7 @@ class PublicAPI:
         while self.torAdder == '':
             clientAPI._core.refreshFirstStartVars()
             self.torAdder = clientAPI._core.hsAddress
-            time.sleep(1)
+            time.sleep(0.1)
         self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None, handler_class=FDSafeHandler)
         self.httpServer.serve_forever()
 
@@ -248,9 +251,6 @@ class API:
             This initilization defines all of the API entry points and handlers for the endpoints and errors
             This also saves the used host (random localhost IP address) to the data folder in host.txt
         '''
-        # assert isinstance(onionrInst, onionr.Onionr)
-        # configure logger and stuff
-        onionr.Onionr.setupConfig('data/', self = self)
 
         self.debug = debug
         self._core = onionrInst.onionrCore
@@ -262,7 +262,7 @@ class API:
         self.bindPort = bindPort
 
         # Be extremely mindful of this. These are endpoints available without a password
-        self.whitelistEndpoints = ('site', 'www', 'onionrhome', 'board', 'boardContent', 'sharedContent', 'mail', 'mailindex')
+        self.whitelistEndpoints = ('site', 'www', 'onionrhome', 'board', 'boardContent', 'sharedContent', 'mail', 'mailindex', 'friends', 'friendsindex')
 
         self.clientToken = config.get('client.webpassword')
         self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
@@ -276,6 +276,9 @@ class API:
         self.pluginResponses = {} # Responses for plugin endpoints
         self.queueResponse = {}
         onionrInst.setClientAPIInst(self)
+        app.register_blueprint(friendsapi.friends)
+        app.register_blueprint(simplecache.simplecache)
+        httpapi.load_plugin_blueprints(app)
 
         @app.before_request
         def validateRequest():
@@ -287,8 +290,10 @@ class API:
                 return
             try:
                 if not hmac.compare_digest(request.headers['token'], self.clientToken):
+                    if not hmac.compare_digest(request.form['token'], self.clientToken):
                         abort(403)
             except KeyError:
+                if not hmac.compare_digest(request.form['token'], self.clientToken):
                     abort(403)
 
         @app.after_request
@@ -316,6 +321,14 @@ class API:
         def loadMailIndex():
             return send_from_directory('static-data/www/mail/', 'index.html')
         
+        @app.route('/friends/<path:path>', endpoint='friends')
+        def loadContacts(path):
+            return send_from_directory('static-data/www/friends/', path)
+
+        @app.route('/friends/', endpoint='friendsindex')
+        def loadContacts():
+            return send_from_directory('static-data/www/friends/', 'index.html')
+
         @app.route('/board/<path:path>', endpoint='boardContent')
         def boardContent(path):
             return send_from_directory('static-data/www/board/', path)
@@ -406,14 +419,16 @@ class API:
             if self._core._utils.validateHash(bHash):
                 try:
                     resp = Block(bHash).bcontent
+                except onionrexceptions.NoDataAvailable:
+                    abort(404)
                 except TypeError:
                     pass
                 try:
                     resp = base64.b64decode(resp)
                 except:
                     pass
-            if resp == 'Not Found':
+            if resp == 'Not Found' or not resp:
-                abourt(404)
+                abort(404)
             return Response(resp)
 
         @app.route('/waitforshare/<name>', methods=['post'])
@@ -512,7 +527,6 @@ class API:
                 data = subpath.split('/')
                 if len(data) > 1:
                     plName = data[0]
-
                     events.event('pluginRequest', {'name': plName, 'path': subpath, 'pluginResponse': pluginResponseCode, 'postData': postData}, onionr=onionrInst)
                     while True:
                         try:

onionr/communicator.py

@@ -19,13 +19,15 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid
+import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid, binascii
-import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
-import onionrdaemontools, onionrsockets, onionr, onionrproofs
-import binascii
 from dependencies import secrets
-from defusedxml import minidom
 from utils import networkmerger
+import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
+from communicatorutils import onionrdaemontools
+import onionrsockets, onionr, onionrproofs
+from communicatorutils import onionrcommunicatortimers, proxypicker
+
+OnionrCommunicatorTimers = onionrcommunicatortimers.OnionrCommunicatorTimers
 
 config.reload()
 class OnionrCommunicatorDaemon:
@@ -44,10 +46,6 @@ class OnionrCommunicatorDaemon:
         self.proxyPort = proxyPort
         self._core = onionrInst.onionrCore
 
-        # initialize NIST beacon salt and time
-        self.nistSaltTimestamp = 0
-        self.powSalt = 0
-
         self.blocksToUpload = []
 
         # loop time.sleep delay in seconds
@@ -171,7 +169,8 @@ class OnionrCommunicatorDaemon:
             # Validate new peers are good format and not already in queue
             invalid = []
             for x in newPeers:
-                if not self._core._utils.validateID(x) or x in self.newPeers:
+                x = x.strip()
+                if not self._core._utils.validateID(x) or x in self.newPeers or x == self._core.hsAddress:
                     invalid.append(x)
             for x in invalid:
                 newPeers.remove(x)
@@ -431,16 +430,18 @@ class OnionrCommunicatorDaemon:
         for address in peerList:
             if not config.get('tor.v3onions') and len(address) == 62:
                 continue
+            if address == self._core.hsAddress:
+                continue
             if len(address) == 0 or address in tried or address in self.onlinePeers or address in self.cooldownPeer:
                 continue
             if self.shutdown:
                 return
             if self.peerAction(address, 'ping') == 'pong!':
-                logger.info('Connected to ' + address)
                 time.sleep(0.1)
                 if address not in mainPeerList:
                     networkmerger.mergeAdders(address, self._core)
                 if address not in self.onlinePeers:
+                    logger.info('Connected to ' + address)
                     self.onlinePeers.append(address)
                     self.connectTimes[address] = self._core._utils.getEpoch()
                 retData = address
@@ -487,7 +488,7 @@ class OnionrCommunicatorDaemon:
                 score = str(self.getPeerProfileInstance(i).score)
                 logger.info(i + ', score: ' + score)
 
-    def peerAction(self, peer, action, data=''):
+    def peerAction(self, peer, action, data='', returnHeaders=False):
         '''Perform a get request to a peer'''
         if len(peer) == 0:
             return False
@@ -511,7 +512,7 @@ class OnionrCommunicatorDaemon:
         else:
             self._core.setAddressInfo(peer, 'lastConnect', self._core._utils.getEpoch())
             self.getPeerProfileInstance(peer).addScore(1)
-        return retData
+        return retData # If returnHeaders, returns tuple of data, headers. if not, just data string
 
     def getPeerProfileInstance(self, peer):
         '''Gets a peer profile instance from the list of profiles, by address name'''
@@ -603,11 +604,7 @@ class OnionrCommunicatorDaemon:
                     triedPeers.append(peer)
                     url = 'http://' + peer + '/upload'
                     data = {'block': block.Block(bl).getRaw()}
-                    proxyType = ''
+                    proxyType = proxypicker.pick_proxy(peer)
-                    if peer.endswith('.onion'):
-                        proxyType = 'tor'
-                    elif peer.endswith('.i2p'):
-                        proxyType = 'i2p'
                     logger.info("Uploading block to " + peer)
                     if not self._core._utils.doPostRequest(url, data=data, proxyType=proxyType) == False:
                         self._core._utils.localCommand('waitforshare/' + bl, post=True)
@@ -644,48 +641,5 @@ class OnionrCommunicatorDaemon:
 
         self.decrementThreadCount('runCheck')
 
-class OnionrCommunicatorTimers:
-    def __init__(self, daemonInstance, timerFunction, frequency, makeThread=True, threadAmount=1, maxThreads=5, requiresPeer=False):
-        self.timerFunction = timerFunction
-        self.frequency = frequency
-        self.threadAmount = threadAmount
-        self.makeThread = makeThread
-        self.requiresPeer = requiresPeer
-        self.daemonInstance = daemonInstance
-        self.maxThreads = maxThreads
-        self._core = self.daemonInstance._core
-
-        self.daemonInstance.timers.append(self)
-        self.count = 0
-
-    def processTimer(self):
-
-        # mark how many instances of a thread we have (decremented at thread end)
-        try:
-            self.daemonInstance.threadCounts[self.timerFunction.__name__]
-        except KeyError:
-            self.daemonInstance.threadCounts[self.timerFunction.__name__] = 0
-
-        # execute thread if it is time, and we are not missing *required* online peer
-        if self.count == self.frequency:
-            try:
-                if self.requiresPeer and len(self.daemonInstance.onlinePeers) == 0:
-                    raise onionrexceptions.OnlinePeerNeeded
-            except onionrexceptions.OnlinePeerNeeded:
-                pass
-            else:
-                if self.makeThread:
-                    for i in range(self.threadAmount):
-                        if self.daemonInstance.threadCounts[self.timerFunction.__name__] >= self.maxThreads:
-                            logger.debug('%s is currently using the maximum number of threads, not starting another.' % self.timerFunction.__name__)
-                        else:
-                            self.daemonInstance.threadCounts[self.timerFunction.__name__] += 1
-                            newThread = threading.Thread(target=self.timerFunction)
-                            newThread.start()
-                else:
-                    self.timerFunction()
-            self.count = -1 # negative 1 because its incremented at bottom
-        self.count += 1
-
 def startCommunicator(onionrInst, proxyPort):
     OnionrCommunicatorDaemon(onionrInst, proxyPort)

onionr/communicatorutils/onionrcommunicatortimers.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file contains timer control for the communicator
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import threading, onionrexceptions, logger
+class OnionrCommunicatorTimers:
+    def __init__(self, daemonInstance, timerFunction, frequency, makeThread=True, threadAmount=1, maxThreads=5, requiresPeer=False):
+        self.timerFunction = timerFunction
+        self.frequency = frequency
+        self.threadAmount = threadAmount
+        self.makeThread = makeThread
+        self.requiresPeer = requiresPeer
+        self.daemonInstance = daemonInstance
+        self.maxThreads = maxThreads
+        self._core = self.daemonInstance._core
+
+        self.daemonInstance.timers.append(self)
+        self.count = 0
+
+    def processTimer(self):
+
+        # mark how many instances of a thread we have (decremented at thread end)
+        try:
+            self.daemonInstance.threadCounts[self.timerFunction.__name__]
+        except KeyError:
+            self.daemonInstance.threadCounts[self.timerFunction.__name__] = 0
+
+        # execute thread if it is time, and we are not missing *required* online peer
+        if self.count == self.frequency and not self.daemonInstance.shutdown:
+            try:
+                if self.requiresPeer and len(self.daemonInstance.onlinePeers) == 0:
+                    raise onionrexceptions.OnlinePeerNeeded
+            except onionrexceptions.OnlinePeerNeeded:
+                pass
+            else:
+                if self.makeThread:
+                    for i in range(self.threadAmount):
+                        if self.daemonInstance.threadCounts[self.timerFunction.__name__] >= self.maxThreads:
+                            logger.debug('%s is currently using the maximum number of threads, not starting another.' % self.timerFunction.__name__)
+                        else:
+                            self.daemonInstance.threadCounts[self.timerFunction.__name__] += 1
+                            newThread = threading.Thread(target=self.timerFunction)
+                            newThread.start()
+                else:
+                    self.timerFunction()
+            self.count = -1 # negative 1 because its incremented at bottom
+        self.count += 1

onionr/communicatorutils/onionrdaemontools.py

@@ -30,16 +30,22 @@ class DaemonTools:
     '''
     def __init__(self, daemon):
             self.daemon = daemon
+            self.announceProgress = {}
             self.announceCache = {}
 
     def announceNode(self):
         '''Announce our node to our peers'''
         retData = False
         announceFail = False
+        
+        # Do not let announceCache get too large
+        if len(self.announceCache) >= 10000:
+            self.announceCache.popitem()
+
         if self.daemon._core.config.get('general.security_level', 0) == 0:
             # Announce to random online peers
             for i in self.daemon.onlinePeers:
-                if not i in self.announceCache:
+                if not i in self.announceCache and not i in self.announceProgress:
                     peer = i
                     break
             else:
@@ -66,7 +72,9 @@ class DaemonTools:
                 elif len(existingRand) > 0:
                     data['random'] = existingRand
                 else:
+                    self.announceProgress[peer] = True
                     proof = onionrproofs.DataPOW(combinedNodes, forceDifficulty=4)
+                    del self.announceProgress[peer]
                     try:
                         data['random'] = base64.b64encode(proof.waitForResult()[1])
                     except TypeError:
@@ -89,6 +97,7 @@ class DaemonTools:
         '''Check if we are connected to the internet or not when we can't connect to any peers'''
         if len(self.daemon.onlinePeers) == 0:
             if not netutils.checkNetwork(self.daemon._core._utils, torPort=self.daemon.proxyPort):
+                if not self.daemon.shutdown:
                     logger.warn('Network check failed, are you connected to the internet?')
                 self.daemon.isOnline = False
             else:
@@ -197,7 +206,7 @@ class DaemonTools:
         fakePeer = ''
         chance = 10
         if secrets.randbelow(chance) == (chance - 1):
-            fakePeer = self.daemon._core._crypto.generatePubKey()[0]
+            fakePeer = 'OVPCZLOXD6DC5JHX4EQ3PSOGAZ3T24F75HQLIUZSDSMYPEOXCPFA===='
             data = secrets.token_hex(secrets.randbelow(500) + 1)
             self.daemon._core.insertBlock(data, header='pm', encryptType='asym', asymPeer=fakePeer, meta={'subject': 'foo'})
         self.daemon.decrementThreadCount('insertDeniableBlock')

onionr/communicatorutils/proxypicker.py

@@ -0,0 +1,25 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    Just picks a proxy
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+def pick_proxy(peer_address):
+    if peer_address.endswith('.onion'):
+        return 'tor'
+    elif peer_address.endswith('.i2p'):
+        return 'i2p'

onionr/core.py

@@ -19,11 +19,11 @@
 '''
 import sqlite3, os, sys, time, math, base64, tarfile, nacl, logger, json, netcontroller, math, config, uuid
 from onionrblockapi import Block
-
+import deadsimplekv as simplekv
 import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions
 import onionrblacklist
 from onionrusers import onionrusers
-import dbcreator, onionrstorage, serializeddata
+import dbcreator, onionrstorage, serializeddata, subprocesspow
 from etc import onionrvalues
 
 if sys.version_info < (3, 6):
@@ -65,6 +65,7 @@ class Core:
             self.dataNonceFile = self.dataDir + 'block-nonces.dat'
             self.dbCreate = dbcreator.DBCreator(self)
             self.forwardKeysFile = self.dataDir + 'forward-keys.db'
+            self.keyStore = simplekv.DeadSimpleKV(self.dataDir + 'cachedstorage.dat', refresh_seconds=5)
 
             # Socket data, defined here because of multithreading constraints with gevent
             self.killSockets = False
@@ -105,7 +106,6 @@ class Core:
                 logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation)
 
             self._utils = onionrutils.OnionrUtils(self)
-            self.blockCache = onionrstorage.BlockCache()
             # Initialize the crypto object
             self._crypto = onionrcrypto.OnionrCrypto(self)
             self._blacklist = onionrblacklist.OnionrBlackList(self)
@@ -121,7 +121,6 @@ class Core:
         '''
             Hack to refresh some vars which may not be set on first start
         '''
-
         if os.path.exists(self.dataDir + '/hs/hostname'):
             with open(self.dataDir + '/hs/hostname', 'r') as hs:
                 self.hsAddress = hs.read().strip()
@@ -468,14 +467,6 @@ class Core:
             except TypeError:
                 pass
 
-        if getPow:
-            try:
-                peerList.append(self._crypto.pubKey + '-' + self._crypto.pubKeyPowToken)
-            except TypeError:
-                pass
-        else:
-            peerList.append(self._crypto.pubKey)
-
         conn.close()
 
         return peerList
@@ -597,10 +588,6 @@ class Core:
         conn = sqlite3.connect(self.blockDB, timeout=30)
         c = conn.cursor()
 
-        # if unsaved:
-        #     execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
-        # else:
-        #     execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;'
         execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
         args = (dateRec,)
         rows = list()
@@ -702,6 +689,8 @@ class Core:
             return False
         retData = False
 
+        createTime = self._utils.getRoundedEpoch()
+
         # check nonce
         dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data))
         try:
@@ -719,10 +708,7 @@ class Core:
         data = str(data)
         plaintext = data
         plaintextMeta = {}
-
+        plaintextPeer = asymPeer
-        # Convert asym peer human readable key to base32 if set
-        if ' ' in asymPeer.strip():
-            asymPeer = self._utils.convertHumanReadableID(asymPeer)
 
         retData = ''
         signature = ''
@@ -745,6 +731,7 @@ class Core:
             pass
 
         if encryptType == 'asym':
+            meta['rply'] = createTime # Duplicate the time in encrypted messages to prevent replays
             if not disableForward and sign and asymPeer != self._crypto.pubKey:
                 try:
                     forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data)
@@ -792,7 +779,7 @@ class Core:
         metadata['meta'] = jsonMeta
         metadata['sig'] = signature
         metadata['signer'] = signer
-        metadata['time'] = self._utils.getRoundedEpoch()
+        metadata['time'] = createTime
 
         # ensure expire is integer and of sane length
         if type(expire) is not type(None):
@@ -800,8 +787,7 @@ class Core:
             metadata['expire'] = expire
 
         # send block data (and metadata) to POW module to get tokenized block data
-        proof = onionrproofs.POW(metadata, data)
+        payload = subprocesspow.SubprocessPOW(data, metadata, self).start()
-        payload = proof.waitForResult()
         if payload != False:
             try:
                 retData = self.setData(payload)
@@ -817,6 +803,9 @@ class Core:
                 self.daemonQueueAdd('uploadBlock', retData)
 
         if retData != False:
+            if plaintextPeer == 'OVPCZLOXD6DC5JHX4EQ3PSOGAZ3T24F75HQLIUZSDSMYPEOXCPFA====':
+                events.event('insertdeniable', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': self._utils.bytesToStr(asymPeer)}, onionr = self.onionrInst, threaded = True)
+            else:
                 events.event('insertblock', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': self._utils.bytesToStr(asymPeer)}, onionr = self.onionrInst, threaded = True)
         return retData
 

onionr/httpapi/__init__.py

@@ -0,0 +1,29 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file registers plugin's flask blueprints for the client http server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import onionrplugins
+
+def load_plugin_blueprints(flaskapp):
+    '''Iterate enabled plugins and load any http endpoints they have'''
+    for plugin in onionrplugins.get_enabled_plugins():
+        plugin = onionrplugins.get_plugin(plugin)
+        try:
+            flaskapp.register_blueprint(getattr(plugin, 'flask_blueprint'))
+        except AttributeError:
+            pass

onionr/httpapi/friendsapi/__init__.py

@@ -0,0 +1,56 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file creates http endpoints for friend management
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import core, json
+from onionrusers import contactmanager
+from flask import Blueprint, Response, request, abort, redirect
+
+friends = Blueprint('friends', __name__)
+
+@friends.route('/friends/list')
+def list_friends():
+    pubkey_list = {}
+    friend_list = contactmanager.ContactManager.list_friends(core.Core())
+    for friend in friend_list:
+        pubkey_list[friend.publicKey] = {'name': friend.get_info('name')}
+    return json.dumps(pubkey_list)
+
+@friends.route('/friends/add/<pubkey>', methods=['POST'])
+def add_friend(pubkey):
+    contactmanager.ContactManager(core.Core(), pubkey, saveUser=True).setTrust(1)
+    return redirect(request.referrer + '#' + request.form['token'])
+
+@friends.route('/friends/remove/<pubkey>', methods=['POST'])
+def remove_friend(pubkey):
+    contactmanager.ContactManager(core.Core(), pubkey).setTrust(0)
+    return redirect(request.referrer + '#' + request.form['token'])
+
+@friends.route('/friends/setinfo/<pubkey>/<key>', methods=['POST'])
+def set_info(pubkey, key):
+    data = request.form['data']
+    contactmanager.ContactManager(core.Core(), pubkey).set_info(key, data)
+    return redirect(request.referrer + '#' + request.form['token'])
+
+@friends.route('/friends/getinfo/<pubkey>/<key>')
+def get_info(pubkey, key):
+    retData = contactmanager.ContactManager(core.Core(), pubkey).get_info(key)
+    if retData is None:
+        abort(404)
+    else:
+        return retData

onionr/httpapi/simplecache/__init__.py

@@ -0,0 +1,31 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This file creates http endpoints for friend management
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import core
+from flask import Blueprint, Response, request, abort
+
+simplecache = Blueprint('simplecache', __name__)
+
+@simplecache.route('/get/<key>')
+def get_key(key):
+    return
+
+@simplecache.route('/set/<key>', methods=['POST'])
+def set_key(key):
+    return

onionr/netcontroller.py

@@ -147,7 +147,8 @@ HiddenServicePort 80 ''' + self.apiServerIP + ''':''' + str(self.hsPort)
             torVersion = subprocess.Popen([self.torBinary, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             for line in iter(torVersion.stdout.readline, b''):
                 if 'Tor 0.2.' in line.decode():
-                    logger.warn("Running 0.2.x Tor series, no support for v3 onion peers")
+                    logger.error('Tor 0.3+ required')
+                    sys.exit(1)
                     break
             torVersion.kill()
 
@@ -162,7 +163,7 @@ HiddenServicePort 80 ''' + self.apiServerIP + ''':''' + str(self.hsPort)
                 logger.fatal('Failed to start Tor. Maybe a stray instance of Tor used by Onionr is still running?')
                 return False
         except KeyboardInterrupt:
-            logger.fatal('Got keyboard interrupt.', timestamp = false, level = logger.LEVEL_IMPORTANT)
+            logger.fatal('Got keyboard interrupt.', timestamp = False, level = logger.LEVEL_IMPORTANT)
             return False
         
         logger.debug('Finished starting Tor.', timestamp=True)

onionr/onionrblockapi.py

@@ -26,7 +26,7 @@ class Block:
     blockCacheOrder = list() # NEVER write your own code that writes to this!
     blockCache = dict() # should never be accessed directly, look at Block.getCache()
 
-    def __init__(self, hash = None, core = None, type = None, content = None, expire=None, decrypt=False):
+    def __init__(self, hash = None, core = None, type = None, content = None, expire=None, decrypt=False, bypassReplayCheck=False):
         # take from arguments
         # sometimes people input a bytes object instead of str in `hash`
         if (not hash is None) and isinstance(hash, bytes):
@@ -37,6 +37,7 @@ class Block:
         self.btype = type
         self.bcontent = content
         self.expire = expire
+        self.bypassReplayCheck = bypassReplayCheck
 
         # initialize variables
         self.valid = True
@@ -84,6 +85,20 @@ class Block:
                 self.signer = core._crypto.pubKeyDecrypt(self.signer, encodedData=encodedData)
                 self.bheader['signer'] = self.signer.decode()
                 self.signedData =  json.dumps(self.bmetadata) + self.bcontent.decode()
+
+                # Check for replay attacks
+                try:
+                    if self.core._utils.getEpoch() - self.core.getBlockDate(self.hash) < 60:
+                        assert self.core._crypto.replayTimestampValidation(self.bmetadata['rply'])
+                except (AssertionError, KeyError) as e:
+                    if not self.bypassReplayCheck:
+                        # Zero out variables to prevent reading of replays
+                        self.bmetadata = {}
+                        self.signer = ''
+                        self.bheader['signer'] = ''
+                        self.signedData = ''
+                        self.signature = ''
+                        raise onionrexceptions.ReplayAttack('Signature is too old. possible replay attack')
                 try:
                     assert self.bmetadata['forwardEnc'] is True
                 except (AssertionError, KeyError) as e:
@@ -97,6 +112,8 @@ class Block:
             except nacl.exceptions.CryptoError:
                 pass
                 #logger.debug('Could not decrypt block. Either invalid key or corrupted data')
+            except onionrexceptions.ReplayAttack:
+                logger.warn('%s is possibly a replay attack' % (self.hash,))
             else:
                 retData = True
                 self.decrypted = True
@@ -139,32 +156,10 @@ class Block:
 
             # import from file
             if blockdata is None:
+                try:
                     blockdata = onionrstorage.getData(self.core, self.getHash()).decode()
-                '''
+                except AttributeError:
-                
+                    raise onionrexceptions.NoDataAvailable('Block does not exist')
-                filelocation = file
-
-                readfile = True
-
-                if filelocation is None:
-                    if self.getHash() is None:
-                        return False
-                    elif self.getHash() in Block.getCache():
-                        # get the block from cache, if it's in it
-                        blockdata = Block.getCache(self.getHash())
-                        readfile = False
-
-                    # read from file if it's still None
-                    if blockdata is None:
-                        filelocation = self.core.dataDir + 'blocks/%s.dat' % self.getHash()
-
-                if readfile:
-                    blockdata = onionrstorage.getData(self.core, self.getHash()).decode()
-                    #with open(filelocation, 'rb') as f:
-                    #blockdata = f.read().decode()
-
-                self.blockFile = filelocation
-                '''
             else:
                 self.blockFile = None
             # parse block
@@ -200,11 +195,11 @@ class Block:
 
             return True
         except Exception as e:
-            logger.error('Failed to parse block %s.' % self.getHash(), error = e, timestamp = False)
+            logger.warn('Failed to parse block %s.' % self.getHash(), error = e, timestamp = False)
 
             # if block can't be parsed, it's a waste of precious space. Throw it away.
             if not self.delete():
-                logger.error('Failed to delete invalid block %s.' % self.getHash(), error = e)
+                logger.warn('Failed to delete invalid block %s.' % self.getHash(), error = e)
             else:
                 logger.debug('Deleted invalid block %s.' % self.getHash(), timestamp = False)
 

onionr/onionrcommands/__init__.py

@@ -0,0 +1,171 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This module defines commands for CLI usage
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import webbrowser, sys
+import logger
+from . import pubkeymanager, onionrstatistics, daemonlaunch, filecommands, plugincommands, keyadders
+
+def show_help(o_inst, command):
+
+    helpmenu = o_inst.getHelp()
+
+    if command is None and len(sys.argv) >= 3:
+        for cmd in sys.argv[2:]:
+            o_inst.showHelp(cmd)
+    elif not command is None:
+        if command.lower() in helpmenu:
+            logger.info(logger.colors.bold + command  + logger.colors.reset + logger.colors.fg.blue + ' : ' + logger.colors.reset +  helpmenu[command.lower()], timestamp = False)
+        else:
+            logger.warn(logger.colors.bold + command  + logger.colors.reset + logger.colors.fg.blue + ' : ' + logger.colors.reset + 'No help menu entry was found', timestamp = False)
+    else:
+        o_inst.version(0)
+        for command, helpmessage in helpmenu.items():
+            o_inst.showHelp(command)
+
+def open_home(o_inst):
+    try:
+        url = o_inst.onionrUtils.getClientAPIServer()
+    except FileNotFoundError:
+        logger.error('Onionr seems to not be running (could not get api host)')
+    else:
+        url = 'http://%s/#%s' % (url, o_inst.onionrCore.config.get('client.webpassword'))
+        print('If Onionr does not open automatically, use this URL:', url)
+        webbrowser.open_new_tab(url)
+
+def get_commands(onionr_inst):
+    return {'': onionr_inst.showHelpSuggestion,
+    'help': onionr_inst.showHelp,
+    'version': onionr_inst.version,
+    'config': onionr_inst.configure,
+    'start': onionr_inst.start,
+    'stop': onionr_inst.killDaemon,
+    'status': onionr_inst.showStats,
+    'statistics': onionr_inst.showStats,
+    'stats': onionr_inst.showStats,
+    'details' : onionr_inst.showDetails,
+    'detail' : onionr_inst.showDetails,
+    'show-details' : onionr_inst.showDetails,
+    'show-detail' : onionr_inst.showDetails,
+    'showdetails' : onionr_inst.showDetails,
+    'showdetail' : onionr_inst.showDetails,
+    'get-details' : onionr_inst.showDetails,
+    'get-detail' : onionr_inst.showDetails,
+    'getdetails' : onionr_inst.showDetails,
+    'getdetail' : onionr_inst.showDetails,
+
+    'enable-plugin': onionr_inst.enablePlugin,
+    'enplugin': onionr_inst.enablePlugin,
+    'enableplugin': onionr_inst.enablePlugin,
+    'enmod': onionr_inst.enablePlugin,
+    'disable-plugin': onionr_inst.disablePlugin,
+    'displugin': onionr_inst.disablePlugin,
+    'disableplugin': onionr_inst.disablePlugin,
+    'dismod': onionr_inst.disablePlugin,
+    'reload-plugin': onionr_inst.reloadPlugin,
+    'reloadplugin': onionr_inst.reloadPlugin,
+    'reload-plugins': onionr_inst.reloadPlugin,
+    'reloadplugins': onionr_inst.reloadPlugin,
+    'create-plugin': onionr_inst.createPlugin,
+    'createplugin': onionr_inst.createPlugin,
+    'plugin-create': onionr_inst.createPlugin,
+
+    'listkeys': onionr_inst.listKeys,
+    'list-keys': onionr_inst.listKeys,
+
+    'addpeer': onionr_inst.addPeer,
+    'add-peer': onionr_inst.addPeer,
+    'add-address': onionr_inst.addAddress,
+    'add-addr': onionr_inst.addAddress,
+    'addaddr': onionr_inst.addAddress,
+    'addaddress': onionr_inst.addAddress,
+    'list-peers': onionr_inst.listPeers,
+
+    'blacklist-block': onionr_inst.banBlock,
+
+    'add-file': onionr_inst.addFile,
+    'addfile': onionr_inst.addFile,
+    'addhtml': onionr_inst.addWebpage,
+    'add-html': onionr_inst.addWebpage,
+    'add-site': onionr_inst.addWebpage,
+    'addsite': onionr_inst.addWebpage,
+
+    'openhome': onionr_inst.openHome,
+    'open-home': onionr_inst.openHome,
+
+    'export-block': onionr_inst.exportBlock,
+    'exportblock': onionr_inst.exportBlock,
+
+    'get-file': onionr_inst.getFile,
+    'getfile': onionr_inst.getFile,
+
+    'listconn': onionr_inst.listConn,
+    'list-conn': onionr_inst.listConn,
+
+    'import-blocks': onionr_inst.onionrUtils.importNewBlocks,
+    'importblocks': onionr_inst.onionrUtils.importNewBlocks,
+
+    'introduce': onionr_inst.onionrCore.introduceNode,
+    'pex': onionr_inst.doPEX,
+
+    'getpassword': onionr_inst.printWebPassword,
+    'get-password': onionr_inst.printWebPassword,
+    'getpwd': onionr_inst.printWebPassword,
+    'get-pwd': onionr_inst.printWebPassword,
+    'getpass': onionr_inst.printWebPassword,
+    'get-pass': onionr_inst.printWebPassword,
+    'getpasswd': onionr_inst.printWebPassword,
+    'get-passwd': onionr_inst.printWebPassword,
+
+    'friend': onionr_inst.friendCmd,
+    'addid': onionr_inst.addID,
+    'add-id': onionr_inst.addID,
+    'change-id': onionr_inst.changeID
+    }
+
+cmd_help = {
+    'help': 'Displays this Onionr help menu',
+    'version': 'Displays the Onionr version',
+    'config': 'Configures something and adds it to the file',
+
+    'start': 'Starts the Onionr daemon',
+    'stop': 'Stops the Onionr daemon',
+
+    'stats': 'Displays node statistics',
+    'details': 'Displays the web password, public key, and human readable public key',
+
+    'enable-plugin': 'Enables and starts a plugin',
+    'disable-plugin': 'Disables and stops a plugin',
+    'reload-plugin': 'Reloads a plugin',
+    'create-plugin': 'Creates directory structure for a plugin',
+
+    'add-peer': 'Adds a peer to database',
+    'list-peers': 'Displays a list of peers',
+    'add-file': 'Create an Onionr block from a file',
+    'get-file': 'Get a file from Onionr blocks',
+    'import-blocks': 'import blocks from the disk (Onionr is transport-agnostic!)',
+    'listconn': 'list connected peers',
+    'pex': 'exchange addresses with peers (done automatically)',
+    'blacklist-block': 'deletes a block by hash and permanently removes it from your node',
+    'introduce': 'Introduce your node to the public Onionr network',
+    'friend': '[add|remove] [public key/id]',
+    'add-id': 'Generate a new ID (key pair)',
+    'change-id': 'Change active ID',
+    'open-home': 'Open your node\'s home/info screen'
+        }

onionr/onionrcommands/daemonlaunch.py

@@ -0,0 +1,142 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    launch the api server and communicator
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import os, time, sys, platform, sqlite3
+from threading import Thread
+import onionr, api, logger, communicator
+import onionrevents as events
+from netcontroller import NetController
+def daemon(o_inst):
+    '''
+        Starts the Onionr communication daemon
+    '''
+
+    # remove runcheck if it exists
+    if os.path.isfile('data/.runcheck'):
+        logger.debug('Runcheck file found on daemon start, deleting in advance.')
+        os.remove('data/.runcheck')
+
+    Thread(target=api.API, args=(o_inst, o_inst.debug, onionr.API_VERSION)).start()
+    Thread(target=api.PublicAPI, args=[o_inst.getClientApi()]).start()
+    try:
+        time.sleep(0)
+    except KeyboardInterrupt:
+        logger.debug('Got keyboard interrupt, shutting down...')
+        time.sleep(1)
+        o_inst.onionrUtils.localCommand('shutdown')
+
+    apiHost = ''
+    while apiHost == '':
+        try:
+            with open(o_inst.onionrCore.publicApiHostFile, 'r') as hostFile:
+                apiHost = hostFile.read()
+        except FileNotFoundError:
+            pass
+        time.sleep(0.5)
+    onionr.Onionr.setupConfig('data/', self = o_inst)
+
+    if o_inst._developmentMode:
+        logger.warn('DEVELOPMENT MODE ENABLED (NOT RECOMMENDED)', timestamp = False)
+    net = NetController(o_inst.onionrCore.config.get('client.public.port', 59497), apiServerIP=apiHost)
+    logger.debug('Tor is starting...')
+    if not net.startTor():
+        o_inst.onionrUtils.localCommand('shutdown')
+        sys.exit(1)
+    if len(net.myID) > 0 and o_inst.onionrCore.config.get('general.security_level') == 0:
+        logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID))
+    else:
+        logger.debug('.onion service disabled')
+    logger.debug('Using public key: %s' % (logger.colors.underline + o_inst.onionrCore._crypto.pubKey))
+    time.sleep(1)
+
+    o_inst.onionrCore.torPort = net.socksPort
+    communicatorThread = Thread(target=communicator.startCommunicator, args=(o_inst, str(net.socksPort)))
+    communicatorThread.start()
+    
+    while o_inst.communicatorInst is None:
+        time.sleep(0.1)
+
+    # print nice header thing :)
+    if o_inst.onionrCore.config.get('general.display_header', True):
+        o_inst.header()
+
+    # print out debug info
+    o_inst.version(verbosity = 5, function = logger.debug)
+    logger.debug('Python version %s' % platform.python_version())
+
+    logger.debug('Started communicator.')
+
+    events.event('daemon_start', onionr = o_inst)
+    try:
+        while True:
+            time.sleep(3)
+            # Debug to print out used FDs (regular and net)
+            #proc = psutil.Process()
+            #print('api-files:',proc.open_files(), len(psutil.net_connections()))
+            # Break if communicator process ends, so we don't have left over processes
+            if o_inst.communicatorInst.shutdown:
+                break
+            if o_inst.killed:
+                break # Break out if sigterm for clean exit
+    except KeyboardInterrupt:
+        pass
+    finally:
+        o_inst.onionrCore.daemonQueueAdd('shutdown')
+        o_inst.onionrUtils.localCommand('shutdown')
+    net.killTor()
+    time.sleep(3)
+    o_inst.deleteRunFiles()
+    return
+
+def kill_daemon(o_inst):
+    '''
+        Shutdown the Onionr daemon
+    '''
+
+    logger.warn('Stopping the running daemon...', timestamp = False)
+    try:
+        events.event('daemon_stop', onionr = o_inst)
+        net = NetController(o_inst.onionrCore.config.get('client.port', 59496))
+        try:
+            o_inst.onionrCore.daemonQueueAdd('shutdown')
+        except sqlite3.OperationalError:
+            pass
+
+        net.killTor()
+    except Exception as e:
+        logger.error('Failed to shutdown daemon.', error = e, timestamp = False)
+    return
+
+def start(o_inst, input = False, override = False):
+    if os.path.exists('.onionr-lock') and not override:
+        logger.fatal('Cannot start. Daemon is already running, or it did not exit cleanly.\n(if you are sure that there is not a daemon running, delete .onionr-lock & try again).')
+    else:
+        if not o_inst.debug and not o_inst._developmentMode:
+            lockFile = open('.onionr-lock', 'w')
+            lockFile.write('')
+            lockFile.close()
+        o_inst.running = True
+        o_inst.daemon()
+        o_inst.running = False
+        if not o_inst.debug and not o_inst._developmentMode:
+            try:
+                os.remove('.onionr-lock')
+            except FileNotFoundError:
+                pass

onionr/onionrcommands/filecommands.py

@@ -0,0 +1,49 @@
+import base64, sys, os
+import logger
+from onionrblockapi import Block
+def add_file(o_inst, singleBlock=False, blockType='bin'):
+    '''
+        Adds a file to the onionr network
+    '''
+
+    if len(sys.argv) >= 3:
+        filename = sys.argv[2]
+        contents = None
+
+        if not os.path.exists(filename):
+            logger.error('That file does not exist. Improper path (specify full path)?')
+            return
+        logger.info('Adding file... this might take a long time.')
+        try:
+            with open(filename, 'rb') as singleFile:
+                blockhash = o_inst.onionrCore.insertBlock(base64.b64encode(singleFile.read()), header=blockType)
+            if len(blockhash) > 0:
+                logger.info('File %s saved in block %s' % (filename, blockhash))
+        except:
+            logger.error('Failed to save file in block.', timestamp = False)
+    else:
+        logger.error('%s add-file <filename>' % sys.argv[0], timestamp = False)
+
+def getFile(o_inst):
+    '''
+        Get a file from onionr blocks
+    '''
+    try:
+        fileName = sys.argv[2]
+        bHash = sys.argv[3]
+    except IndexError:
+        logger.error("Syntax %s %s" % (sys.argv[0], '/path/to/filename <blockhash>'))
+    else:
+        logger.info(fileName)
+
+        contents = None
+        if os.path.exists(fileName):
+            logger.error("File already exists")
+            return
+        if not o_inst.onionrUtils.validateHash(bHash):
+            logger.error('Block hash is invalid')
+            return
+
+        with open(fileName, 'wb') as myFile:
+            myFile.write(base64.b64decode(Block(bHash, core=o_inst.onionrCore).bcontent))
+    return

onionr/onionrcommands/keyadders.py

@@ -0,0 +1,49 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    add keys (transport and pubkey)
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys
+import logger
+def add_peer(o_inst):
+    try:
+        newPeer = sys.argv[2]
+    except IndexError:
+        pass
+    else:
+        if o_inst.onionrUtils.hasKey(newPeer):
+            logger.info('We already have that key')
+            return
+        logger.info("Adding peer: " + logger.colors.underline + newPeer)
+        try:
+            if o_inst.onionrCore.addPeer(newPeer):
+                logger.info('Successfully added key')
+        except AssertionError:
+            logger.error('Failed to add key')
+
+def add_address(o_inst):
+    try:
+        newAddress = sys.argv[2]
+        newAddress = newAddress.replace('http:', '').replace('/', '')
+    except IndexError:
+        pass
+    else:
+        logger.info("Adding address: " + logger.colors.underline + newAddress)
+        if o_inst.onionrCore.addAddress(newAddress):
+            logger.info("Successfully added address.")
+        else:
+            logger.warn("Unable to add address.")

onionr/onionrcommands/onionrstatistics.py

@@ -0,0 +1,110 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This module defines commands to show stats/details about the local node
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import os, uuid, time
+import logger, onionrutils
+from onionrblockapi import Block
+import onionr
+
+def show_stats(o_inst):
+    try:
+        # define stats messages here
+        totalBlocks = len(o_inst.onionrCore.getBlockList())
+        signedBlocks = len(Block.getBlocks(signed = True))
+        messages = {
+            # info about local client
+            'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if o_inst.onionrUtils.isCommunicatorRunning(timeout = 9) else logger.colors.fg.red + 'Offline'),
+
+            # file and folder size stats
+            'div1' : True, # this creates a solid line across the screen, a div
+            'Total Block Size' : onionrutils.humanSize(onionrutils.size(o_inst.dataDir + 'blocks/')),
+            'Total Plugin Size' : onionrutils.humanSize(onionrutils.size(o_inst.dataDir + 'plugins/')),
+            'Log File Size' : onionrutils.humanSize(onionrutils.size(o_inst.dataDir + 'output.log')),
+
+            # count stats
+            'div2' : True,
+            'Known Peers' : str(len(o_inst.onionrCore.listPeers()) - 1),
+            'Enabled Plugins' : str(len(o_inst.onionrCore.config.get('plugins.enabled', list()))) + ' / ' + str(len(os.listdir(o_inst.dataDir + 'plugins/'))),
+            'Stored Blocks' : str(totalBlocks),
+            'Percent Blocks Signed' : str(round(100 * signedBlocks / max(totalBlocks, 1), 2)) + '%'
+        }
+
+        # color configuration
+        colors = {
+            'title' : logger.colors.bold,
+            'key' : logger.colors.fg.lightgreen,
+            'val' : logger.colors.fg.green,
+            'border' : logger.colors.fg.lightblue,
+
+            'reset' : logger.colors.reset
+        }
+
+        # pre-processing
+        maxlength = 0
+        width = o_inst.getConsoleWidth()
+        for key, val in messages.items():
+            if not (type(val) is bool and val is True):
+                maxlength = max(len(key), maxlength)
+        prewidth = maxlength + len(' | ')
+        groupsize = width - prewidth - len('[+] ')
+
+        # generate stats table
+        logger.info(colors['title'] + 'Onionr v%s Statistics' % onionr.ONIONR_VERSION + colors['reset'])
+        logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'])
+        for key, val in messages.items():
+            if not (type(val) is bool and val is True):
+                val = [str(val)[i:i + groupsize] for i in range(0, len(str(val)), groupsize)]
+
+                logger.info(colors['key'] + str(key).rjust(maxlength) + colors['reset'] + colors['border'] + ' | ' + colors['reset'] + colors['val'] + str(val.pop(0)) + colors['reset'])
+
+                for value in val:
+                    logger.info(' ' * maxlength + colors['border'] + ' | ' + colors['reset'] + colors['val'] + str(value) + colors['reset'])
+            else:
+                logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'])
+        logger.info(colors['border'] + '-' * (maxlength + 1) + '+' + colors['reset'])
+    except Exception as e:
+        logger.error('Failed to generate statistics table.', error = e, timestamp = False)
+
+def show_details(o_inst):
+    details = {
+        'Node Address' : o_inst.get_hostname(),
+        'Web Password' : o_inst.getWebPassword(),
+        'Public Key' : o_inst.onionrCore._crypto.pubKey,
+        'Human-readable Public Key' : o_inst.onionrCore._utils.getHumanReadableID()
+    }
+
+    for detail in details:
+        logger.info('%s%s: \n%s%s\n' % (logger.colors.fg.lightgreen, detail, logger.colors.fg.green, details[detail]), sensitive = True)
+
+def show_peers(o_inst):
+    randID = str(uuid.uuid4())
+    o_inst.onionrCore.daemonQueueAdd('connectedPeers', responseID=randID)
+    while True:
+        try:
+            time.sleep(3)
+            peers = o_inst.onionrCore.daemonQueueGetResponse(randID)
+        except KeyboardInterrupt:
+            break
+        if not type(peers) is None:
+            if peers not in ('', 'failure', None):
+                if peers != False:
+                    print(peers)
+                else:
+                    print('Daemon probably not running. Unable to list connected peers.')
+                break

onionr/onionrcommands/plugincommands.py

@@ -0,0 +1,88 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    plugin CLI commands
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import sys
+import logger, onionrplugins as plugins
+
+def enable_plugin(o_inst):
+    if len(sys.argv) >= 3:
+        plugin_name = sys.argv[2]
+        logger.info('Enabling plugin "%s"...' % plugin_name)
+        plugins.enable(plugin_name, o_inst)
+    else:
+        logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]))
+
+def disable_plugin(o_inst):
+
+    if len(sys.argv) >= 3:
+        plugin_name = sys.argv[2]
+        logger.info('Disabling plugin "%s"...' % plugin_name)
+        plugins.disable(plugin_name, o_inst)
+    else:
+        logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]))
+
+def reload_plugin(o_inst):
+    '''
+        Reloads (stops and starts) all plugins, or the given plugin
+    '''
+
+    if len(sys.argv) >= 3:
+        plugin_name = sys.argv[2]
+        logger.info('Reloading plugin "%s"...' % plugin_name)
+        plugins.stop(plugin_name, o_inst)
+        plugins.start(plugin_name, o_inst)
+    else:
+        logger.info('Reloading all plugins...')
+        plugins.reload(o_inst)
+
+
+def create_plugin(o_inst):
+    '''
+        Creates the directory structure for a plugin name
+    '''
+
+    if len(sys.argv) >= 3:
+        try:
+            plugin_name = re.sub('[^0-9a-zA-Z_]+', '', str(sys.argv[2]).lower())
+
+            if not plugins.exists(plugin_name):
+                logger.info('Creating plugin "%s"...' % plugin_name)
+
+                os.makedirs(plugins.get_plugins_folder(plugin_name))
+                with open(plugins.get_plugins_folder(plugin_name) + '/main.py', 'a') as main:
+                    contents = ''
+                    with open('static-data/default_plugin.py', 'rb') as file:
+                        contents = file.read().decode()
+
+                    # TODO: Fix $user. os.getlogin() is   B U G G Y
+                    main.write(contents.replace('$user', 'some random developer').replace('$date', datetime.datetime.now().strftime('%Y-%m-%d')).replace('$name', plugin_name))
+
+                with open(plugins.get_plugins_folder(plugin_name) + '/info.json', 'a') as main:
+                    main.write(json.dumps({'author' : 'anonymous', 'description' : 'the default description of the plugin', 'version' : '1.0'}))
+
+                logger.info('Enabling plugin "%s"...' % plugin_name)
+                plugins.enable(plugin_name, o_inst)
+            else:
+                logger.warn('Cannot create plugin directory structure; plugin "%s" exists.' % plugin_name)
+
+        except Exception as e:
+            logger.error('Failed to create plugin directory structure.', e)
+    else:
+        logger.info('%s %s <plugin>' % (sys.argv[0], sys.argv[1]))

onionr/onionrcommands/pubkeymanager.py

@@ -0,0 +1,101 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This module defines ID-related CLI commands
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+import sys
+import logger
+from onionrusers import onionrusers
+def add_ID(o_inst):
+    try:
+        sys.argv[2]
+        assert sys.argv[2] == 'true'
+    except (IndexError, AssertionError) as e:
+        newID = o_inst.onionrCore._crypto.keyManager.addKey()[0]
+    else:
+        logger.warn('Deterministic keys require random and long passphrases.')
+        logger.warn('If a good passphrase is not used, your key can be easily stolen.')
+        logger.warn('You should use a series of hard to guess words, see this for reference: https://www.xkcd.com/936/')
+        pass1 = getpass.getpass(prompt='Enter at least %s characters: ' % (o_inst.onionrCore._crypto.deterministicRequirement,))
+        pass2 = getpass.getpass(prompt='Confirm entry: ')
+        if o_inst.onionrCore._crypto.safeCompare(pass1, pass2):
+            try:
+                logger.info('Generating deterministic key. This can take a while.')
+                newID, privKey = o_inst.onionrCore._crypto.generateDeterministic(pass1)
+            except onionrexceptions.PasswordStrengthError:
+                logger.error('Must use at least 25 characters.')
+                sys.exit(1)
+        else:
+            logger.error('Passwords do not match.')
+            sys.exit(1)
+        o_inst.onionrCore._crypto.keyManager.addKey(pubKey=newID, 
+        privKey=privKey)
+    logger.info('Added ID: %s' % (o_inst.onionrUtils.bytesToStr(newID),))
+
+def change_ID(o_inst):
+    try:
+        key = sys.argv[2]
+    except IndexError:
+        logger.error('Specify pubkey to use')
+    else:
+        if o_inst.onionrUtils.validatePubKey(key):
+            if key in o_inst.onionrCore._crypto.keyManager.getPubkeyList():
+                o_inst.onionrCore.config.set('general.public_key', key)
+                o_inst.onionrCore.config.save()
+                logger.info('Set active key to: %s' % (key,))
+                logger.info('Restart Onionr if it is running.')
+            else:
+                logger.error('That key does not exist')
+        else:
+            logger.error('Invalid key %s' % (key,))
+
+def friend_command(o_inst):
+    friend = ''
+    try:
+        # Get the friend command
+        action = sys.argv[2]
+    except IndexError:
+        logger.info('Syntax: friend add/remove/list [address]')
+    else:
+        action = action.lower()
+        if action == 'list':
+            # List out peers marked as our friend
+            for friend in onionrusers.OnionrUser.list_friends(o_inst.onionrCore):
+                logger.info(friend.publicKey + ' - ' + friend.getName())
+        elif action in ('add', 'remove'):
+            try:
+                friend = sys.argv[3]
+                if not o_inst.onionrUtils.validatePubKey(friend):
+                    raise onionrexceptions.InvalidPubkey('Public key is invalid')
+                if friend not in o_inst.onionrCore.listPeers():
+                    raise onionrexceptions.KeyNotKnown
+                friend = onionrusers.OnionrUser(o_inst.onionrCore, friend)
+            except IndexError:
+                logger.error('Friend ID is required.')
+            except onionrexceptions.KeyNotKnown:
+                o_inst.onionrCore.addPeer(friend)
+                friend = onionrusers.OnionrUser(o_inst.onionrCore, friend)
+            finally:
+                if action == 'add':
+                    friend.setTrust(1)
+                    logger.info('Added %s as friend.' % (friend.publicKey,))
+                else:
+                    friend.setTrust(0)
+                    logger.info('Removed %s as friend.' % (friend.publicKey,))
+        else:
+            logger.info('Syntax: friend add/remove/list [address]')

onionr/onionrcrypto.py

@@ -264,6 +264,13 @@ class OnionrCrypto:
 
         return retData
 
+    @staticmethod
+    def replayTimestampValidation(timestamp):
+        if core.Core()._utils.getEpoch() - int(timestamp) > 2419200:
+            return False
+        else:
+            return True
+
     @staticmethod
     def safeCompare(one, two):
         # Do encode here to avoid spawning core

onionr/onionrevents.py

@@ -67,7 +67,6 @@ def call(plugin, event_name, data = None, pluginapi = None):
 
             return True
         except Exception as e:
-            logger.debug(str(e))
             return False
     else:
         return True

onionr/onionrexceptions.py

@@ -45,6 +45,9 @@ class PasswordStrengthError(Exception):
 
 # block exceptions
 
+class ReplayAttack(Exception):
+    pass
+
 class DifficultyTooLarge(Exception):
     pass
 

onionr/onionrplugins.py

@@ -59,7 +59,6 @@ def reload(onionr = None, stop_event = True):
 
     return False
 
-
 def enable(name, onionr = None, start_event = True):
     '''
         Enables a plugin
@@ -73,6 +72,8 @@ def enable(name, onionr = None, start_event = True):
             try:
                 events.call(get_plugin(name), 'enable', onionr)
             except ImportError: # Was getting import error on Gitlab CI test "data"
+                # NOTE: If you are experiencing issues with plugins not being enabled, it might be this resulting from an error in the module
+                # can happen inconsistenly (especially between versions)
                 return False
             else:
                 enabled_plugins.append(name)

onionr/onionrproofs.py

@@ -17,10 +17,8 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-
+import multiprocessing, nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, sys, base64, json
-import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json
+import core, onionrutils, config, logger, onionrblockapi
-import core, onionrutils, config
-import onionrblockapi
 
 def getDifficultyModifier(coreOrUtilsInst=None):
     '''Accepts a core or utils instance returns 
@@ -101,7 +99,7 @@ def hashMeetsDifficulty(h):
         return False
 
 class DataPOW:
-    def __init__(self, data, forceDifficulty=0, threadCount = 5):
+    def __init__(self, data, forceDifficulty=0, threadCount = 1):
         self.foundHash = False
         self.difficulty = 0
         self.data = data
@@ -200,7 +198,7 @@ class DataPOW:
         return result
 
 class POW:
-    def __init__(self, metadata, data, threadCount = 5, forceDifficulty=0, coreInst=None):
+    def __init__(self, metadata, data, threadCount = 1, forceDifficulty=0, coreInst=None):
         self.foundHash = False
         self.difficulty = 0
         self.data = data
@@ -246,6 +244,7 @@ class POW:
         answer = ''
         hbCount = 0
         nonce = int(binascii.hexlify(nacl.utils.random(2)), 16)
+        startNonce = nonce
         while self.hashing:
             #token = nacl.hash.blake2b(rand + self.data).decode()
             self.metadata['powRandomToken'] = nonce
@@ -260,6 +259,7 @@ class POW:
                 self.hashing = False
                 iFound = True
                 self.result = payload
+                print('count', nonce - startNonce)
                 break
             nonce += 1
                 

onionr/onionrstorage.py

@@ -21,13 +21,6 @@ import core, sys, sqlite3, os, dbcreator
 
 DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
 
-class BlockCache:
-    def __init__(self):
-        self.blocks = {}
-    def cleanCache(self):
-        while sys.getsizeof(self.blocks) > 100000000:
-            self.blocks.pop(list(self.blocks.keys())[0])
-
 def dbCreate(coreInst):
     try:
         dbcreator.DBCreator(coreInst).createBlockDataDB()
@@ -84,7 +77,6 @@ def store(coreInst, data, blockHash=''):
     else:
         with open('%s/%s.dat' % (coreInst.blockDataLocation, blockHash), 'wb') as blockFile:
             blockFile.write(data)
-    coreInst.blockCache.cleanCache()
 
 def getData(coreInst, bHash):
     assert isinstance(coreInst, core.Core)

onionr/onionrusers/onionrusers.py

@@ -17,7 +17,7 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import onionrblockapi, logger, onionrexceptions, json, sqlite3
+import onionrblockapi, logger, onionrexceptions, json, sqlite3, time
 import nacl.exceptions
 
 def deleteExpiredKeys(coreInst):
@@ -76,11 +76,11 @@ class OnionrUser:
         return retData
 
     def encrypt(self, data):
-        encrypted = coreInst._crypto.pubKeyEncrypt(data, self.publicKey, encodedData=True)
+        encrypted = self._core._crypto.pubKeyEncrypt(data, self.publicKey, encodedData=True)
         return encrypted
 
     def decrypt(self, data):
-        decrypted = coreInst._crypto.pubKeyDecrypt(data, self.publicKey, encodedData=True)
+        decrypted = self._core._crypto.pubKeyDecrypt(data, self.publicKey, encodedData=True)
         return decrypted
 
     def forwardEncrypt(self, data):
@@ -112,7 +112,8 @@ class OnionrUser:
         conn = sqlite3.connect(self._core.peerDB, timeout=10)
         c = conn.cursor()
 
-        for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE peerKey = ? ORDER BY date DESC", (self.publicKey,)):
+        # TODO: account for keys created at the same time (same epoch)
+        for row in c.execute("SELECT forwardKey, max(DATE) FROM forwardKeys WHERE peerKey = ?", (self.publicKey,)):
             key = row[0]
             break
 
@@ -126,9 +127,8 @@ class OnionrUser:
         c = conn.cursor()
         keyList = []
 
-        for row in c.execute("SELECT forwardKey FROM forwardKeys WHERE peerKey = ? ORDER BY date DESC", (self.publicKey,)):
+        for row in c.execute("SELECT forwardKey, date FROM forwardKeys WHERE peerKey = ? ORDER BY date DESC", (self.publicKey,)):
-            key = row[0]
+            keyList.append((row[0], row[1]))
-            keyList.append(key)
 
         conn.commit()
         conn.close()
@@ -175,32 +175,36 @@ class OnionrUser:
 
     def addForwardKey(self, newKey, expire=604800):
         if not self._core._utils.validatePubKey(newKey):
+            # Do not add if something went wrong with the key
             raise onionrexceptions.InvalidPubkey(newKey)
-        if newKey in self._getForwardKeys():
+
-            return False
-        # Add a forward secrecy key for the peer
         conn = sqlite3.connect(self._core.peerDB, timeout=10)
         c = conn.cursor()
+
+        # Get the time we're inserting the key at
+        timeInsert = self._core._utils.getEpoch()
+
+        # Look at our current keys for duplicate key data or time
+        for entry in self._getForwardKeys():
+            if entry[0] == newKey:
+                return False
+            if entry[1] == timeInsert:
+                timeInsert += 1
+                time.sleep(1) # Sleep if our time is the same in order to prevent duplicate time records
+
+        # Add a forward secrecy key for the peer
         # Prepare the insert
-        time = self._core._utils.getEpoch()
+        command = (self.publicKey, newKey, timeInsert, timeInsert + expire)
-        command = (self.publicKey, newKey, time, time + expire)
 
         c.execute("INSERT INTO forwardKeys VALUES(?, ?, ?, ?);", command)
 
         conn.commit()
         conn.close()
-        return
+        return True
     
-    def findAndSetID(self):
+    @classmethod
-        '''Find any info about the user from existing blocks and cache it to their DB entry'''
+    def list_friends(cls, coreInst):
-        infoBlocks = []
+        friendList = []
-        for bHash in self._core.getBlocksByType('userInfo'):
+        for x in coreInst.listPeers(trust=1):
-            block = onionrblockapi.Block(bHash, core=self._core)
+            friendList.append(cls(coreInst, x))
-            if block.signer == self.publicKey:
+        return list(friendList)
-                if block.verifySig():
-                    newName = block.getMetadata('name')
-                    if newName.isalnum():
-                        logger.info('%s is now using the name %s.' % (self.publicKey, self._core._utils.escapeAnsi(newName)))
-                        self._core.setPeerInfo(self.publicKey, 'name', newName)
-            else:
-                raise onionrexceptions.InvalidPubkey

onionr/onionrutils.py

@@ -278,12 +278,19 @@ class OnionrUtils:
                         break
                     if (self.getEpoch() - metadata[i]) > maxAge:
                         logger.warn('Block is outdated: %s' % (metadata[i],))
+                        break
                 elif i == 'expire':
                     try:
                         assert int(metadata[i]) > self.getEpoch()
                     except AssertionError:
                         logger.warn('Block is expired')
                         break
+                elif i == 'encryptType':
+                    try:
+                        assert metadata[i] in ('asym', 'sym', '')
+                    except AssertionError:
+                        logger.warn('Invalid encryption mode')
+                        break
             else:
                 # if metadata loop gets no errors, it does not break, therefore metadata is valid
                 # make sure we do not have another block with the same data content (prevent data duplication and replay attacks)
@@ -408,12 +415,14 @@ class OnionrUtils:
             This function is intended to scan for new blocks ON THE DISK and import them
         '''
         blockList = self._core.getBlockList()
+        exist = False
         if scanDir == '':
             scanDir = self._core.blockDataLocation
         if not scanDir.endswith('/'):
             scanDir += '/'
         for block in glob.glob(scanDir + "*.dat"):
             if block.replace(scanDir, '').replace('.dat', '') not in blockList:
+                exist = True
                 logger.info('Found new block on dist %s' % block)
                 with open(block, 'rb') as newBlock:
                     block = block.replace(scanDir, '').replace('.dat', '')
@@ -423,6 +432,8 @@ class OnionrUtils:
                         self._core._utils.processBlockMetadata(block)
                     else:
                         logger.warn('Failed to verify hash for %s' % block)
+        if not exist:
+            print('No blocks found to import')
 
     def progressBar(self, value = 0, endvalue = 100, width = None):
         '''
@@ -469,7 +480,7 @@ class OnionrUtils:
             retData = False
         return retData
 
-    def doGetRequest(self, url, port=0, proxyType='tor', ignoreAPI=False):
+    def doGetRequest(self, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False):
         '''
         Do a get request through a local tor or i2p instance
         '''
@@ -509,6 +520,9 @@ class OnionrUtils:
             if not 'ConnectTimeoutError' in str(e) and not 'Request rejected or failed' in str(e):
                 logger.debug('Error: %s' % str(e))
             retData = False
+        if returnHeaders:
+            return (retData, response_headers)
+        else:
             return retData
 
     def strToBytes(self, data):

onionr/setupconfig.py

@@ -0,0 +1,69 @@
+import os, json
+import config, logger
+
+def setup_config(dataDir, o_inst = None):
+    data_exists = os.path.exists(dataDir)
+
+    if not data_exists:
+        os.mkdir(dataDir)
+
+    if os.path.exists('static-data/default_config.json'):
+        # this is the default config, it will be overwritten if a config file already exists. Else, it saves it
+        with open('static-data/default_config.json', 'r') as configReadIn:
+            config.set_config(json.loads(configReadIn.read())) 
+    else:
+        # the default config file doesn't exist, try hardcoded config
+        logger.warn('Default configuration file does not exist, switching to hardcoded fallback configuration!')
+        config.set_config({'dev_mode': True, 'log': {'file': {'output': True, 'path': dataDir + 'output.log'}, 'console': {'output': True, 'color': True}}})
+    if not data_exists:
+        config.save()
+    config.reload() # this will read the configuration file into memory
+
+    settings = 0b000
+    if config.get('log.console.color', True):
+        settings = settings | logger.USE_ANSI
+    if config.get('log.console.output', True):
+        settings = settings | logger.OUTPUT_TO_CONSOLE
+    if config.get('log.file.output', True):
+        settings = settings | logger.OUTPUT_TO_FILE
+    logger.set_settings(settings)
+
+    if not o_inst is None:
+        if str(config.get('general.dev_mode', True)).lower() == 'true':
+            o_inst._developmentMode = True
+            logger.set_level(logger.LEVEL_DEBUG)
+        else:
+            o_inst._developmentMode = False
+            logger.set_level(logger.LEVEL_INFO)
+
+    verbosity = str(config.get('log.verbosity', 'default')).lower().strip()
+    if not verbosity in ['default', 'null', 'none', 'nil']:
+        map = {
+            str(logger.LEVEL_DEBUG) : logger.LEVEL_DEBUG,
+            'verbose' : logger.LEVEL_DEBUG,
+            'debug' : logger.LEVEL_DEBUG,
+            str(logger.LEVEL_INFO) : logger.LEVEL_INFO,
+            'info' : logger.LEVEL_INFO,
+            'information' : logger.LEVEL_INFO,
+            str(logger.LEVEL_WARN) : logger.LEVEL_WARN,
+            'warn' : logger.LEVEL_WARN,
+            'warning' : logger.LEVEL_WARN,
+            'warnings' : logger.LEVEL_WARN,
+            str(logger.LEVEL_ERROR) : logger.LEVEL_ERROR,
+            'err' : logger.LEVEL_ERROR,
+            'error' : logger.LEVEL_ERROR,
+            'errors' : logger.LEVEL_ERROR,
+            str(logger.LEVEL_FATAL) : logger.LEVEL_FATAL,
+            'fatal' : logger.LEVEL_FATAL,
+            str(logger.LEVEL_IMPORTANT) : logger.LEVEL_IMPORTANT,
+            'silent' : logger.LEVEL_IMPORTANT,
+            'quiet' : logger.LEVEL_IMPORTANT,
+            'important' : logger.LEVEL_IMPORTANT
+        }
+
+        if verbosity in map:
+            logger.set_level(map[verbosity])
+        else:
+            logger.warn('Verbosity level %s is not valid, using default verbosity.' % verbosity)
+
+    return data_exists

onionr/static-data/bootstrap-nodes.txt

@@ -1 +0,0 @@
-dd3llxdp5q6ak3zmmicoy3jnodmroouv2xr7whkygiwp3rl7nf23gdad.onion

onionr/static-data/default-plugins/cliui/main.py

@@ -19,8 +19,10 @@
 '''
 
 # Imports some useful libraries
-import logger, config, threading, time, uuid, subprocess, sys
+import threading, time, uuid, subprocess, sys
+import config, logger
 from onionrblockapi import Block
+import onionrplugins
 
 plugin_name = 'cliui'
 PLUGIN_VERSION = '0.0.1'
@@ -29,7 +31,11 @@ class OnionrCLIUI:
     def __init__(self, apiInst):
         self.api = apiInst
         self.myCore = apiInst.get_core()
-        return
+        self.shutdown = False
+        self.running = 'undetermined'
+        enabled = onionrplugins.get_enabled_plugins()
+        self.mail_enabled = 'pms' in enabled
+        self.flow_enabled = 'flow' in enabled
 
     def subCommand(self, command, args=None):
             try:
@@ -42,26 +48,27 @@ class OnionrCLIUI:
             except KeyboardInterrupt:
                 pass
     
+    def isRunning(self):
+        while not self.shutdown:
+            if self.myCore._utils.localCommand('ping', maxWait=5) == 'pong!':
+                self.running = 'Yes'
+            else:
+                self.running = 'No'
+            time.sleep(5)
+
     def refresh(self):
             print('\n' * 80 + logger.colors.reset)
 
     def start(self):
         '''Main CLI UI interface menu'''
         showMenu = True
-        isOnline = 'No'
-        firstRun = True
         choice = ''
-        if self.myCore._utils.localCommand('ping', maxWait=10) == 'pong!':
+        threading.Thread(target=self.isRunning).start()
-            firstRun = False
 
         while showMenu:
-            if self.myCore._utils.localCommand('ping', maxWait=2) == 'pong!':
+            print('Onionr\n------')
-                isOnline = "Yes"
+            print('''Daemon Running: ''' + self.running + '''
-            else:
+1. Flow (Anonymous public shout box, use at your own risk)
-                isOnline = "No"
-
-            print('''Daemon Running: ''' + isOnline + '''
-1. Flow (Anonymous public chat, use at your own risk)
 2. Mail (Secure email-like service)
 3. File Sharing
 4. Quit (Does not shutdown daemon)
@@ -72,21 +79,27 @@ class OnionrCLIUI:
                 choice = "quit"
 
             if choice in ("flow", "1"):
+                if self.flow_enabled:
                     self.subCommand("flow")
+                else:
+                    print('Plugin not enabled')
             elif choice in ("2", "mail"):
+                if self.mail_enabled:
                     self.subCommand("mail")
+                else:
+                    print('Plugin not enabled')
             elif choice in ("3", "file sharing", "file"):
                 filename = input("Enter full path to file: ").strip()
                 self.subCommand("addfile", filename)
             elif choice in ("4", "quit"):
                 showMenu = False
+                self.shutdown = True
             elif choice == "":
                 pass
             else:
                 logger.error("Invalid choice")
         return
 
-
 def on_init(api, data = None):
     '''
         This event is called after Onionr is initialized, but before the command

onionr/static-data/default-plugins/contactmanager/info.json

@@ -0,0 +1,5 @@
+{
+    "name" : "contactmanager",
+    "version" : "1.0",
+    "author" : "onionr"
+}

onionr/static-data/default-plugins/contactmanager/main.py

@@ -0,0 +1,39 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    This is an interactive menu-driven CLI interface for Onionr
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+
+plugin_name = 'contactmanager'
+
+class OnionrContactManager:
+    def __init__(self, api):
+        return
+
+def on_init(api, data = None):
+    '''
+        This event is called after Onionr is initialized, but before the command
+        inputted is executed. Could be called when daemon is starting or when
+        just the client is running.
+    '''
+
+    # Doing this makes it so that the other functions can access the api object
+    # by simply referencing the variable `pluginapi`.
+    pluginapi = api
+    ui = OnionrContactManager(api)
+    #api.commands.register('interactive', ui.start)
+    return

onionr/static-data/default-plugins/encrypt/main.py

@@ -19,7 +19,7 @@
 '''
 
 # Imports some useful libraries
-import logger, config, threading, time, readline, datetime, sys, json
+import logger, config, threading, time, datetime, sys, json
 from onionrblockapi import Block
 import onionrexceptions, onionrusers
 import locale

onionr/static-data/default-plugins/pms/loadinbox.py

@@ -0,0 +1,13 @@
+import onionrblockapi
+def load_inbox(myCore):
+    inbox_list = []
+    deleted = myCore.keyStore.get('deleted_mail')
+    if deleted is None:
+        deleted = []
+
+    for blockHash in myCore.getBlocksByType('pm'):
+        block = onionrblockapi.Block(blockHash, core=myCore)
+        block.decrypt()
+        if block.decrypted and blockHash not in deleted:
+            inbox_list.append(blockHash)
+    return inbox_list

onionr/static-data/default-plugins/pms/mailapi.py

@@ -0,0 +1,65 @@
+'''
+    Onionr - P2P Anonymous Storage Network
+
+    HTTP endpoints for mail plugin.
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys, os, json
+from flask import Response, request, redirect, Blueprint, abort
+import core
+from onionrusers import contactmanager
+sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
+import loadinbox, sentboxdb
+
+flask_blueprint = Blueprint('mail', __name__)
+c = core.Core()
+kv = c.keyStore
+
+@flask_blueprint.route('/mail/ping')
+def mail_ping():
+    return 'pong!'
+
+@flask_blueprint.route('/mail/deletemsg/<block>', methods=['POST'])
+def mail_delete(block):
+    if not c._utils.validateHash(block):
+        abort(504)
+    existing = kv.get('deleted_mail')
+    if existing is None:
+        existing = []
+    if block not in existing:
+        existing.append(block)
+    kv.put('deleted_mail', existing)
+    return 'success'
+
+@flask_blueprint.route('/mail/getinbox')
+def list_inbox():
+    return ','.join(loadinbox.load_inbox(c))
+
+@flask_blueprint.route('/mail/getsentbox')
+def list_sentbox():
+    sentbox_list = sentboxdb.SentBox(c).listSent()
+    sentbox_list_copy = list(sentbox_list)
+    deleted = kv.get('deleted_mail')
+    if deleted is None:
+        deleted = []
+    for x in range(len(sentbox_list_copy) - 1):
+        if sentbox_list_copy[x]['hash'] in deleted:
+            x -= 1
+            sentbox_list.pop(x)
+        else:
+            sentbox_list[x]['name'] = contactmanager.ContactManager(c, sentbox_list_copy[x]['peer'], saveUser=False).get_info('name')
+
+    return json.dumps(sentbox_list)

onionr/static-data/default-plugins/pms/main.py

@@ -19,7 +19,7 @@
 '''
 
 # Imports some useful libraries
-import logger, config, threading, time, readline, datetime
+import logger, config, threading, time, datetime
 from onionrblockapi import Block
 import onionrexceptions
 from onionrusers import onionrusers
@@ -27,12 +27,13 @@ import locale, sys, os, json
 
 locale.setlocale(locale.LC_ALL, '')
 
-sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
-import sentboxdb # import after path insert
-
 plugin_name = 'pms'
 PLUGIN_VERSION = '0.0.1'
 
+sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
+import sentboxdb, mailapi, loadinbox # import after path insert
+flask_blueprint = mailapi.flask_blueprint
+
 def draw_border(text):
     #https://stackoverflow.com/a/20757491
     lines = text.splitlines()
@@ -43,7 +44,6 @@ def draw_border(text):
     res.append('└' + '─' * width + '┘')
     return '\n'.join(res)
 
-
 class MailStrings:
     def __init__(self, mailInstance):
         self.mailInstance = mailInstance
@@ -78,7 +78,7 @@ class OnionrMail:
         displayList = []
         subject = ''
 
-        # this could use a lot of memory if someone has recieved a lot of messages
+        # this could use a lot of memory if someone has received a lot of messages
         for blockHash in self.myCore.getBlocksByType('pm'):
             pmBlocks[blockHash] = Block(blockHash, core=self.myCore)
             pmBlocks[blockHash].decrypt()
@@ -191,7 +191,6 @@ class OnionrMail:
                 finally:
                     if choice == '-q':
                         entering = False
-
         return
 
     def get_sent_list(self, display=True):
@@ -294,26 +293,20 @@ class OnionrMail:
                 logger.warn('Invalid choice.')
         return
 
+def add_deleted(keyStore, bHash):
+    existing = keyStore.get('deleted_mail')
+    if existing is None:
+        existing = []
+    else:
+        if bHash in existing:
+            return
+    keyStore.put('deleted_mail', existing.append(bHash))
+
 def on_insertblock(api, data={}):
     sentboxTools = sentboxdb.SentBox(api.get_core())
     meta = json.loads(data['meta'])
     sentboxTools.addToSent(data['hash'], data['peer'], data['content'], meta['subject'])
 
-def on_pluginrequest(api, data=None):
-    resp = ''
-    subject = ''
-    recip = ''
-    message = ''
-    postData = {}
-    blockID = ''
-    sentboxTools = sentboxdb.SentBox(api.get_core())
-    if data['name'] == 'mail':
-        path = data['path']
-        cmd = path.split('/')[1]
-        if cmd == 'sentbox':
-            resp = OnionrMail(api).get_sent_list(display=False)
-    if resp != '':
-        api.get_onionr().clientAPIInst.pluginResponses[data['pluginResponse']] = resp
 
 def on_init(api, data = None):
     '''

onionr/static-data/default-plugins/pms/sentboxdb.py

@@ -25,11 +25,16 @@ class SentBox:
         self.dbLocation = mycore.dataDir + 'sentbox.db'
         if not os.path.exists(self.dbLocation):
             self.createDB()
-        self.conn = sqlite3.connect(self.dbLocation)
-        self.cursor = self.conn.cursor()
         self.core = mycore
         return
     
+    def connect(self):
+        self.conn = sqlite3.connect(self.dbLocation)
+        self.cursor = self.conn.cursor()
+
+    def close(self):
+        self.conn.close()
+
     def createDB(self):
         conn = sqlite3.connect(self.dbLocation)
         cursor = conn.cursor()
@@ -42,22 +47,29 @@ class SentBox:
             );
         ''')
         conn.commit()
+        conn.close()
         return
 
     def listSent(self):
+        self.connect()
         retData = []
         for entry in self.cursor.execute('SELECT * FROM sent;'):
             retData.append({'hash': entry[0], 'peer': entry[1], 'message': entry[2], 'subject': entry[3], 'date': entry[4]})
+        self.close()
         return retData
 
     def addToSent(self, blockID, peer, message, subject=''):
+        self.connect()
         args = (blockID, peer, message, subject, self.core._utils.getEpoch())
         self.cursor.execute('INSERT INTO sent VALUES(?, ?, ?, ?, ?)', args)
         self.conn.commit()
+        self.close()
         return
 
     def removeSent(self, blockID):
+        self.connect()
         args = (blockID,)
         self.cursor.execute('DELETE FROM sent where hash=?', args)
         self.conn.commit()
+        self.close()
         return

onionr/static-data/default_config.json

@@ -8,7 +8,8 @@
         "security_level": 0,
         "max_block_age": 2678400,
         "bypass_tor_check": false,
-        "public_key": ""
+        "public_key": "",
+        "random_bind_ip": true
     },
 
     "www" : {
@@ -48,7 +49,7 @@
         "verbosity" : "default",
 
         "file": {
-            "output": true,
+            "output": false,
             "path": "output.log"
         },
 

onionr/static-data/www/friends/friends.js

@@ -0,0 +1,97 @@
+/*
+    Onionr - P2P Anonymous Storage Network
+
+    This file handles the UI for managing friends/contacts
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+friendListDisplay = document.getElementById('friendList')
+addForm = document.getElementById('addFriend')
+
+function removeFriend(pubkey){
+    post_to_url('/friends/remove/' + pubkey, {'token': webpass})
+}
+
+addForm.onsubmit = function(){
+    var friend = document.getElementsByName('addKey')[0]
+    var alias = document.getElementsByName('data')[0]
+
+    fetch('/friends/add/' + friend.value, {
+        method: 'POST',
+        headers: {
+          "token": webpass
+        }}).then(function(data) {
+            if (alias.value.trim().length > 0){
+                post_to_url('/friends/setinfo/' + friend.value + '/name', {'data': alias.value, 'token': webpass})
+            }
+        })
+
+    return false
+}
+
+fetch('/friends/list', {
+    headers: {
+      "token": webpass
+    }})
+.then((resp) => resp.json()) // Transform the data into json
+.then(function(resp) {
+    var keys = [];
+    for(var k in resp) keys.push(k);
+    console.log(keys)
+    friendListDisplay.innerHTML = 'Click name to view info<br><br>'
+    for (var i = 0; i < keys.length; i++){
+        var peer = keys[i]
+        var name = resp[keys[i]]['name']
+        if (name === null || name === ''){
+            name = peer
+        }
+        var entry = document.createElement('div')
+        var nameText = document.createElement('input')
+        removeButton = document.createElement('button')
+        removeButton.classList.add('friendRemove')
+        removeButton.classList.add('dangerBtn')
+        entry.setAttribute('data-pubkey', peer)
+        removeButton.innerText = 'X'
+        nameText.value = name
+        nameText.readOnly = true
+        nameText.style.fontStyle = "italic"
+        entry.style.paddingTop = '8px'
+        entry.appendChild(removeButton)
+        entry.appendChild(nameText)
+        friendListDisplay.appendChild(entry)
+        entry.onclick = (function(entry, nameText, peer) {return function() {
+            if (nameText.length == 0){
+                nameText = 'Anonymous'
+            }
+            document.getElementById('friendPubkey').value = peer
+            document.getElementById('friendName').innerText = nameText
+            overlay('friendInfo')
+        };})(entry, nameText.value, peer);
+    }
+    // If friend delete buttons are pressed
+
+    var friendRemoveBtns = document.getElementsByClassName('friendRemove')
+
+    for (var x = 0; x < friendRemoveBtns.length; x++){
+        var friendKey = friendRemoveBtns[x].parentElement.getAttribute('data-pubkey')
+        friendRemoveBtns[x].onclick = function(){
+            removeFriend(friendKey)
+        }
+    }
+  })
+
+  document.getElementById('defriend').onclick = function(){
+      removeFriend(document.getElementById('friendPubkey').value)
+  }

onionr/static-data/www/friends/style.css

@@ -0,0 +1,41 @@
+h2, h3{
+    font-family: Arial, Helvetica, sans-serif;
+}
+
+form{
+    border: 1px solid black;
+    border-radius: 5px;
+    padding: 1em;
+    margin-right: 10%;
+}
+form label{
+    display: block;
+    margin-top: 0.5em;
+    margin-bottom: 0.5em;
+}
+
+#friendList{
+    display: inline;
+}
+#friendList span{
+    text-align: center;
+}
+#friendList button{
+    display: inline;
+    margin-right: 10px;
+}
+
+#friendInfo .overlayContent{
+    background-color: lightgray;
+    border: 3px solid black;
+    border-radius: 3px;
+    color: black;
+    font-family: Verdana, Geneva, Tahoma, sans-serif;
+    min-height: 100%;
+    padding: 1em;
+    margin: 1em;
+}
+#defriend{
+    display: block;
+    margin-top: 1em;
+}

onionr/static-data/www/mail/index.html

@@ -16,7 +16,13 @@
         <div class='content'>
             <img class='logo' src='/shared/onionr-icon.png' alt='onionr logo'>
             <span class='logoText'>Onionr Mail ✉️</span>
-            <div>Current Used Identity: <input class='myPub' type='text' readonly> <button class='refresh'>Refresh Page</button></div>
+            <br><br>
+            <div><a href='/' class='idLink'>Home</a> <button class='refresh'>Refresh Page</button></div>
+            <div class='mailPing'>
+                API server either shutdown, has disabled mail, or has experienced a bug.
+            </div>
+            <br>
+            <div>Current Used Identity: <input class='myPub' type='text' readonly></div>
             <br><br>
             <div class="btn-group" id='tabBtns'>
                 <button class='activeTab'>Inbox</button><button>Sentbox</button><button>Send Message</button>
@@ -30,6 +36,9 @@
                     <div>
                         From: <input type='text' id='fromUser' readonly> Signature: <span id='sigValid'></span>
                     </div>
+                    <div>
+                        <button id='replyBtn' class='primaryBtn'>Reply</button>
+                    </div>
                     <div id='signatureValidity'></div>
                     <div id='threadDisplay' class='pre messageContent'>
                     </div>
@@ -45,6 +54,7 @@
             </div>
             <div id='sendMessage' class='overlay'>
                 <div class='overlayContent'>
+                    <label>Select friend: <select id='friendSelect'></select></label>
                     <form method='post' action='/apipoints/mail/send' id='sendForm' enctype="application/x-www-form-urlencoded">
                         <span class='closeOverlay' overlay='sendMessage'></span>
                         To: <input id='draftID' type='text' name='to' placeholder='pubkey' required>

onionr/static-data/www/mail/mail.css

@@ -53,6 +53,11 @@ input{
       margin: 1em;
   }
 
+  .mailPing{
+      display: none;
+      color: orange;
+  }
+
 .danger{
     color: red;
 }
@@ -87,6 +92,17 @@ input{
     color: black;
 }
 
+#replyBtn{
+    margin-top: 1em;
+}
+
+.primaryBtn{
+    border-radius: 3px;
+    padding: 3px;
+    color: black;
+    width: 5%;
+}
+
 .successBtn{
     background-color: #28a745;
     border-radius: 3px;

onionr/static-data/www/mail/mail.js

@@ -24,8 +24,27 @@ threadPlaceholder = document.getElementById('threadPlaceholder')
 tabBtns = document.getElementById('tabBtns')
 threadContent = {}
 myPub = httpGet('/getActivePubkey')
+replyBtn = document.getElementById('replyBtn')
 
-function openThread(bHash, sender, date, sigBool){
+function openReply(bHash){
+    var inbox = document.getElementsByClassName('threadEntry')
+    var entry = ''
+    var friendName = ''
+    var key = ''
+    for(var i = 0; i < inbox.length; i++) {
+        if (inbox[i].getAttribute('data-hash') === bHash){
+            entry = inbox[i]
+        }
+    }
+    if (entry.getAttribute('data-nameSet') == 'true'){
+        document.getElementById('friendSelect').value = entry.getElementsByTagName('input')[0].value
+    }
+    key = entry.getAttribute('data-pubkey')
+    document.getElementById('draftID').value = key
+    setActiveTab('send message')
+}
+
+function openThread(bHash, sender, date, sigBool, pubkey){
     var messageDisplay = document.getElementById('threadDisplay')
     var blockContent = httpGet('/getblockbody/' + bHash)
     document.getElementById('fromUser').value = sender
@@ -38,18 +57,22 @@ function openThread(bHash, sender, date, sigBool){
         sigEl.classList.remove('danger')
     }
     else{
-        sigMsg = 'Bad/no ' + sigMsg + ' (message could be fake)'
+        sigMsg = 'Bad/no ' + sigMsg + ' (message could be impersonating someone)'
         sigEl.classList.add('danger')
+        replyBtn.style.display = 'none'
     }
     sigEl.innerText = sigMsg
     overlay('messageDisplay')
+    replyBtn.onclick = function(){
+        openReply(bHash)
+    }
 }
 
 function setActiveTab(tabName){
     threadPart.innerHTML = ""
     switch(tabName){
         case 'inbox':
-            getInbox()
+            refreshPms()
             break
         case 'sentbox':
             getSentbox()
@@ -60,7 +83,39 @@ function setActiveTab(tabName){
     }
 }
 
-function loadInboxEntrys(bHash){
+function deleteMessage(bHash){
+    fetch('/mail/deletemsg/' + bHash, {
+        "method": "post",
+        headers: {
+            "token": webpass
+        }})
+    .then((resp) => resp.text()) // Transform the data into json
+    .then(function(resp) {
+    })
+}
+
+function mailPing(){
+    fetch('/mail/ping', {
+        "method": "get",
+        headers: {
+            "token": webpass
+        }})
+    .then(function(resp) {
+        var pings = document.getElementsByClassName('mailPing')
+        if (resp.ok){
+            for (var i=0; i < pings.length; i++){
+                pings[i].style.display = 'none';
+            }
+        }
+        else{
+            for (var i=0; i < pings.length; i++){
+                pings[i].style.display = 'block';
+            }
+        }
+    })
+}
+
+function loadInboxEntries(bHash){
     fetch('/getblockheader/' + bHash, {
         headers: {
           "token": webpass
@@ -74,26 +129,31 @@ function loadInboxEntrys(bHash){
         var subjectLine = document.createElement('span')
         var dateStr = document.createElement('span')
         var validSig = document.createElement('span')
+        var deleteBtn = document.createElement('button')
         var humanDate = new Date(0)
         var metadata = resp['metadata']
         humanDate.setUTCSeconds(resp['meta']['time'])
+        validSig.style.display = 'none'
         if (resp['meta']['signer'] != ''){
-            senderInput.value = httpGet('/getHumanReadable/' + resp['meta']['signer'])
+            senderInput.value = httpGet('/friends/getinfo/' + resp['meta']['signer'] + '/name')
         }
-        if (resp['meta']['validSig']){
+        if (! resp['meta']['validSig']){
-            validSig.innerText = 'Signature Validity: Good'
+            validSig.style.display = 'inline'
-        }
-        else{
             validSig.innerText = 'Signature Validity: Bad'
             validSig.style.color = 'red'
         }
+        entry.setAttribute('data-nameSet', true)
         if (senderInput.value == ''){
-            senderInput.value = 'Anonymous'
+            senderInput.value = resp['meta']['signer']
+            entry.setAttribute('data-nameSet', false)
         }
         bHashDisplay.innerText = bHash.substring(0, 10)
-        entry.setAttribute('hash', bHash)
+        entry.setAttribute('data-hash', bHash)
+        entry.setAttribute('data-pubkey', resp['meta']['signer'])
         senderInput.readOnly = true
         dateStr.innerText = humanDate.toString()
+        deleteBtn.innerText = 'X'
+        deleteBtn.classList.add('dangerBtn', 'deleteBtn')
         if (metadata['subject'] === undefined || metadata['subject'] === null) {
             subjectLine.innerText = '()'
         }
@@ -102,15 +162,24 @@ function loadInboxEntrys(bHash){
         }
         //entry.innerHTML = 'sender ' + resp['meta']['signer'] + ' - ' + resp['meta']['time'] 
         threadPart.appendChild(entry)
+        entry.appendChild(deleteBtn)
         entry.appendChild(bHashDisplay)
         entry.appendChild(senderInput)
-        entry.appendChild(validSig)
         entry.appendChild(subjectLine)
         entry.appendChild(dateStr)
+        entry.appendChild(validSig)
         entry.classList.add('threadEntry')
 
-        entry.onclick = function(){
+        entry.onclick = function(event){
-            openThread(entry.getAttribute('hash'), senderInput.value, dateStr.innerText, resp['meta']['validSig'])
+            if (event.target.classList.contains('deleteBtn')){
+                return
+            }
+            openThread(entry.getAttribute('data-hash'), senderInput.value, dateStr.innerText, resp['meta']['validSig'], entry.getAttribute('data-pubkey'))
+        }
+
+        deleteBtn.onclick = function(){
+            entry.parentNode.removeChild(entry);
+            deleteMessage(entry.getAttribute('data-hash'))
         }
         
       }.bind(bHash))
@@ -127,7 +196,7 @@ function getInbox(){
             threadPlaceholder.style.display = 'none'
             showed = true
         }
-        loadInboxEntrys(pms[i])
+        loadInboxEntries(pms[i])
     }
     if (! showed){
         threadPlaceholder.style.display = 'block'
@@ -135,7 +204,7 @@ function getInbox(){
 }
 
 function getSentbox(){
-    fetch('/apipoints/mail/sentbox', {
+    fetch('/mail/getsentbox', {
         headers: {
           "token": webpass
         }})
@@ -143,25 +212,49 @@ function getSentbox(){
     .then(function(resp) {
         var keys = [];
         var entry = document.createElement('div')
-        var entryUsed;
         for(var k in resp) keys.push(k);
+        if (keys.length == 0){
+            threadPart.innerHTML = "nothing to show here yet."
+        }
         for (var i = 0; i < keys.length; i++){
             var entry = document.createElement('div')
-            var obj = resp[i];
+            var obj = resp[i]
             var toLabel = document.createElement('span')
             toLabel.innerText = 'To: '
             var toEl = document.createElement('input')
+            var sentDate = document.createElement('span')
+            var humanDate = new Date(0)
+            humanDate.setUTCSeconds(resp[i]['date'])
             var preview = document.createElement('span')
+            var deleteBtn = document.createElement('button')
+            var message = resp[i]['message']
+            deleteBtn.classList.add('deleteBtn', 'dangerBtn')
+            deleteBtn.innerText = 'X'
             toEl.readOnly = true
-            toEl.value = resp[keys[i]][1]
+            sentDate.innerText = humanDate
-            preview.innerText = '(' + resp[keys[i]][2] + ')'
+            if (resp[i]['name'] == null){
+                toEl.value = resp[i]['peer']
+            }
+            else{
+                toEl.value = resp[i]['name']
+            }
+            preview.innerText = '(' + resp[i]['subject'] + ')'
+            entry.setAttribute('data-hash', resp[i]['hash'])
+            entry.appendChild(deleteBtn)
             entry.appendChild(toLabel)
             entry.appendChild(toEl)
             entry.appendChild(preview)
-            entryUsed = resp[keys[i]]
+            entry.appendChild(sentDate)
-            entry.onclick = function(){
+            entry.onclick = (function(tree, el, msg) {return function() {
                 console.log(resp)
-                showSentboxWindow(toEl.value, entryUsed[0])
+                if (! entry.classList.contains('deleteBtn')){
+                    showSentboxWindow(el.value, msg)
+                }
+            };})(entry, toEl, message);
+            
+            deleteBtn.onclick = function(){
+                entry.parentNode.removeChild(entry);
+                deleteMessage(entry.getAttribute('data-hash'))
             }
             threadPart.appendChild(entry)
         } 
@@ -175,15 +268,17 @@ function showSentboxWindow(to, content){
     overlay('sentboxDisplay')
 }
 
-fetch('/getblocksbytype/pm', {
+function refreshPms(){
+fetch('/mail/getinbox', {
     headers: {
       "token": webpass
     }})
 .then((resp) => resp.text()) // Transform the data into json
 .then(function(data) {
     pms = data.split(',')
-    setActiveTab('inbox')
+    getInbox()
   })
+}
 
 tabBtns.onclick = function(event){
     var children = tabBtns.children
@@ -196,13 +291,12 @@ tabBtns.onclick = function(event){
 }
 
 var idStrings = document.getElementsByClassName('myPub')
-var myHumanReadable = httpGet('/getHumanReadable/' + myPub)
 for (var i = 0; i < idStrings.length; i++){
     if (idStrings[i].tagName.toLowerCase() == 'input'){
-        idStrings[i].value = myHumanReadable
+        idStrings[i].value = myPub
     }
     else{
-        idStrings[i].innerText = myHumanReadable
+        idStrings[i].innerText = myPub
     }
 }
 
@@ -210,9 +304,39 @@ for (var i = 0; i < document.getElementsByClassName('refresh').length; i++){
     document.getElementsByClassName('refresh')[i].style.float = 'right'
 }
 
-for (var i = 0; i < document.getElementsByClassName('closeOverlay').length; i++){
-    document.getElementsByClassName('closeOverlay')[i].onclick = function(e){
-        document.getElementById(e.target.getAttribute('overlay')).style.visibility = 'hidden'
-    }
-}
 
+fetch('/friends/list', {
+    headers: {
+      "token": webpass
+    }})
+.then((resp) => resp.json()) // Transform the data into json
+.then(function(resp) {
+    var friendSelectParent = document.getElementById('friendSelect')
+    var keys = [];
+    var friend
+    for(var k in resp) keys.push(k);
+
+    friendSelectParent.appendChild(document.createElement('option'))
+    for (var i = 0; i < keys.length; i++) {
+        var option = document.createElement("option")
+        var name = resp[keys[i]]['name']
+        option.value = keys[i]
+        if (name.length == 0){
+            option.text = keys[i]
+        }
+        else{
+            option.text = name
+        }
+        friendSelectParent.appendChild(option)
+    }
+
+    for (var i = 0; i < keys.length; i++){
+        
+        //friendSelectParent
+        //alert(resp[keys[i]]['name'])
+    }
+})
+setActiveTab('inbox')
+
+setInterval(function(){mailPing()}, 10000)
+mailPing()

onionr/static-data/www/mail/sendmail.js

@@ -18,11 +18,16 @@
 */
 
 var sendbutton = document.getElementById('sendMail')
+messageContent = document.getElementById('draftText')
+to = document.getElementById('draftID')
+subject = document.getElementById('draftSubject')
+friendPicker = document.getElementById('friendSelect')
 
 function sendMail(to, message, subject){
     //postData = {"postData": '{"to": "' + to + '", "message": "' + message + '"}'} // galaxy brain
     postData = {'message': message, 'to': to, 'type': 'pm', 'encrypt': true, 'meta': JSON.stringify({'subject': subject})}
     postData = JSON.stringify(postData)
+    sendForm.style.display = 'none'
     fetch('/insertblock', {
         method: 'POST',
         body: postData,
@@ -32,14 +37,23 @@ function sendMail(to, message, subject){
         }})
     .then((resp) => resp.text()) // Transform the data into json
     .then(function(data) {
+        sendForm.style.display = 'block'
+        alert('Queued for sending!')
       })
 }
 
-sendForm.onsubmit = function(){
+var friendPicker = document.getElementById('friendSelect')
-    var messageContent = document.getElementById('draftText')
+friendPicker.onchange = function(){
-    var to = document.getElementById('draftID')
+    to.value = friendPicker.value
-    var subject = document.getElementById('draftSubject')
+}
-
+
-    sendMail(to.value, messageContent.value, subject.value)
+sendForm.onsubmit = function(){
-    return false;
+    if (friendPicker.value.trim().length !== 0 && to.value.trim().length !== 0){
+        if (friendPicker.value !== to.value){
+            alert('You have selected both a friend and entered a public key manually.')
+            return false
+        }
+    }
+    sendMail(to.value, messageContent.value, subject.value)
+    return false
 }

onionr/static-data/www/shared/main/style.css

@@ -151,3 +151,31 @@ body{
      content: '❌';
      padding: 5px;
  }
+
+ .btn, .warnBtn, .dangerBtn, .successBtn{
+     padding: 5px;
+     border-radius: 5px;
+     border: 2px solid black;
+ }
+.warnBtn{
+    background-color: orange;
+    color: black;
+}
+.dangerBtn{
+    background-color: #f44336;
+    color: black;
+}
+.successBtn{
+    background-color: #4CAF50;
+    color: black;
+}
+
+.primaryBtn{
+    background-color:#396BAC;    
+}
+
+.openSiteBtn{
+    padding: 5px;
+    border: 1px solid black;
+    border-radius: 5px;
+}

onionr/static-data/www/shared/misc.js

@@ -20,6 +20,25 @@
 webpass = document.location.hash.replace('#', '')
 nowebpass = false
 
+function post_to_url(path, params) {
+
+    var form = document.createElement("form")
+
+    form.setAttribute("method", "POST")
+    form.setAttribute("action", path)
+
+    for(var key in params) {
+        var hiddenField = document.createElement("input")
+        hiddenField.setAttribute("type", "hidden")
+        hiddenField.setAttribute("name", key)
+        hiddenField.setAttribute("value", params[key])
+        form.appendChild(hiddenField)
+    }
+
+    document.body.appendChild(form)
+    form.submit() 
+}
+
 if (typeof webpass == "undefined"){
     webpass = localStorage['webpass']
 }
@@ -67,3 +86,9 @@ for(var i = 0; i < refreshLinks.length; i++) {
         location.reload()
     }
 }
+
+for (var i = 0; i < document.getElementsByClassName('closeOverlay').length; i++){
+    document.getElementsByClassName('closeOverlay')[i].onclick = function(e){
+        document.getElementById(e.target.getAttribute('overlay')).style.visibility = 'hidden'
+    }
+}

onionr/static-data/www/shared/sites.js

@@ -0,0 +1,18 @@
+function checkHex(str) {
+    regexp = /^[0-9a-fA-F]+$/
+    if (regexp.test(str)){
+        return true
+    }
+    return false
+}
+
+document.getElementById('openSite').onclick = function(){
+    var hash = document.getElementById('siteViewer').value
+
+    if (checkHex(hash) && hash.length == 64){
+        window.location.href = '/site/' + hash
+    }
+    else{
+        alert('Invalid site hash')
+    }
+}

onionr/subprocesspow.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+import subprocess, sys, os
+import multiprocessing, threading, time, json, math, binascii
+from multiprocessing import Pipe, Process
+import core, onionrblockapi, config, onionrutils, logger, onionrproofs
+
+class SubprocessPOW:
+    def __init__(self, data, metadata, core_inst=None, subprocCount=None):
+        if core_inst is None:
+            core_inst = core.Core()
+        if subprocCount is None:
+            subprocCount = os.cpu_count()
+        self.subprocCount = subprocCount
+        self.result = ''
+        self.shutdown = False
+        self.core_inst = core_inst
+        self.data = data
+        self.metadata = metadata
+
+        dataLen = len(data) + len(json.dumps(metadata))
+
+        #if forceDifficulty > 0:
+        #    self.difficulty = forceDifficulty
+        #else:
+            # Calculate difficulty. Dumb for now, may use good algorithm in the future.
+        self.difficulty = onionrproofs.getDifficultyForNewBlock(dataLen)
+            
+        try:
+            self.data = self.data.encode()
+        except AttributeError:
+            pass
+        
+        logger.info('Computing POW (difficulty: %s)...' % self.difficulty)
+
+        self.mainHash = '0' * 64
+        self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))]
+        self.shutdown = False
+        self.payload = None
+
+    def start(self):
+        startTime = self.core_inst._utils.getEpoch()
+        for x in range(self.subprocCount):
+            threading.Thread(target=self._spawn_proc).start()
+        while True:
+            if self.payload is None:
+                time.sleep(0.1)
+            else:
+                self.shutdown = True
+                return self.payload
+    
+    def _spawn_proc(self):
+        parent_conn, child_conn = Pipe()
+        p = Process(target=self.do_pow, args=(child_conn,))
+        p.start()
+        p.join()
+        payload = None
+        try:
+            while True:
+                data = parent_conn.recv()
+                if len(data) >= 1:
+                    payload = data
+                    break
+        except KeyboardInterrupt:
+            pass
+        finally:
+            parent_conn.send('shutdown')
+            self.payload = payload
+
+    def do_pow(self, pipe):
+        nonce = int(binascii.hexlify(os.urandom(2)), 16)
+        nonceStart = nonce
+        data = self.data
+        metadata = self.metadata
+        puzzle = self.puzzle
+        difficulty = self.difficulty
+        mcore = core.Core()
+        while True:
+            metadata['powRandomToken'] = nonce
+            payload = json.dumps(metadata).encode() + b'\n' + data
+            token = mcore._crypto.sha3Hash(payload)
+            try:
+                # on some versions, token is bytes
+                token = token.decode()
+            except AttributeError:
+                pass
+            if pipe.poll() and pipe.recv() == 'shutdown':
+                break
+            if puzzle == token[0:difficulty]:
+                pipe.send(payload)
+                break
+            nonce += 1
+        

onionr/tests/test_database_actions.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+import sys, os
+sys.path.append(".")
+import unittest, uuid, sqlite3
+TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
+print("Test directory:", TEST_DIR)
+os.environ["ONIONR_HOME"] = TEST_DIR
+from urllib.request import pathname2url
+import core, onionr
+
+c = core.Core()
+
+class OnionrTests(unittest.TestCase):
+    
+    def test_address_add(self):
+        testAddresses = ['facebookcorewwwi.onion', '56kmnycrvepfarolhnx6t2dvmldfeyg7jdymwgjb7jjzg47u2lqw2sad.onion', '5bvb5ncnfr4dlsfriwczpzcvo65kn7fnnlnt2ln7qvhzna2xaldq.b32.i2p']
+        for address in testAddresses:
+            c.addAddress(address)
+        dbAddresses = c.listAdders()
+        for address in testAddresses:
+            self.assertIn(address, dbAddresses)
+        
+        invalidAddresses = [None, '', '   ', '\t', '\n', ' test ', 24, 'fake.onion', 'fake.b32.i2p']
+        for address in invalidAddresses:
+            try:
+                c.addAddress(address)
+            except TypeError:
+                pass
+        dbAddresses = c.listAdders()
+        for address in invalidAddresses:
+            self.assertNotIn(address, dbAddresses) 
+    
+    def test_address_info(self):
+        adder = 'nytimes3xbfgragh.onion'
+        c.addAddress(adder)
+        self.assertNotEqual(c.getAddressInfo(adder, 'success'), 1000)
+        c.setAddressInfo(adder, 'success', 1000)
+        self.assertEqual(c.getAddressInfo(adder, 'success'), 1000)
+
+unittest.main()

onionr/tests/test_forward_secrecy.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+import sys, os, random
+sys.path.append(".")
+import unittest, uuid
+TEST_DIR_1 = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
+TEST_DIR_2 = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
+import core, onionr, time
+
+import onionrexceptions
+from onionrusers import onionrusers
+from onionrusers import contactmanager
+
+class OnionrForwardSecrecyTests(unittest.TestCase):
+    '''
+        Tests both the onionrusers class and the contactmanager (which inherits it)
+    '''
+
+    def test_forward_encrypt(self):
+        os.environ["ONIONR_HOME"] = TEST_DIR_1
+        o = onionr.Onionr()
+        
+        friend = o.onionrCore._crypto.generatePubKey()
+
+        friendUser = onionrusers.OnionrUser(o.onionrCore, friend[0], saveUser=True)
+
+        for x in range(5):
+            message = 'hello world %s' % (random.randint(1, 1000))
+            forwardKey = friendUser.generateForwardKey()
+
+            fakeForwardPair = o.onionrCore._crypto.generatePubKey()
+
+            self.assertTrue(friendUser.addForwardKey(fakeForwardPair[0]))
+
+            encrypted = friendUser.forwardEncrypt(message)
+
+            decrypted = o.onionrCore._crypto.pubKeyDecrypt(encrypted[0], privkey=fakeForwardPair[1], encodedData=True)
+            self.assertEqual(decrypted, message.encode())
+        return
+
+unittest.main()

onionr/tests/test_highlevelcrypto.py

@@ -1,23 +1,23 @@
 #!/usr/bin/env python3
 import sys, os
 sys.path.append(".")
-import unittest, uuid, hashlib
+import unittest, uuid, hashlib, base64
 import nacl.exceptions
 import nacl.signing, nacl.hash, nacl.encoding
 TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
 print("Test directory:", TEST_DIR)
 os.environ["ONIONR_HOME"] = TEST_DIR
-import core, onionr
+import core, onionr, onionrexceptions
 
 c = core.Core()
 crypto = c._crypto
 class OnionrCryptoTests(unittest.TestCase):
     
     def test_blake2b(self):
-        self.assertTrue(crypto.blake2bHash('test') == crypto.blake2bHash(b'test'))
+        self.assertEqual(crypto.blake2bHash('test'), crypto.blake2bHash(b'test'))
-        self.assertTrue(crypto.blake2bHash(b'test') == crypto.blake2bHash(b'test'))
+        self.assertEqual(crypto.blake2bHash(b'test'), crypto.blake2bHash(b'test'))
 
-        self.assertFalse(crypto.blake2bHash('') == crypto.blake2bHash(b'test'))
+        self.assertNotEqual(crypto.blake2bHash(''), crypto.blake2bHash(b'test'))
         try:
             crypto.blake2bHash(None)
         except nacl.exceptions.TypeError:
@@ -25,14 +25,14 @@ class OnionrCryptoTests(unittest.TestCase):
         else:
             self.assertTrue(False)
         
-        self.assertTrue(nacl.hash.blake2b(b'test') == crypto.blake2bHash(b'test'))
+        self.assertEqual(nacl.hash.blake2b(b'test'), crypto.blake2bHash(b'test'))
     
     def test_sha3256(self):
         hasher = hashlib.sha3_256()
-        self.assertTrue(crypto.sha3Hash('test') == crypto.sha3Hash(b'test'))
+        self.assertEqual(crypto.sha3Hash('test'), crypto.sha3Hash(b'test'))
-        self.assertTrue(crypto.sha3Hash(b'test') == crypto.sha3Hash(b'test'))
+        self.assertEqual(crypto.sha3Hash(b'test'), crypto.sha3Hash(b'test'))
 
-        self.assertFalse(crypto.sha3Hash('') == crypto.sha3Hash(b'test'))
+        self.assertNotEqual(crypto.sha3Hash(''), crypto.sha3Hash(b'test'))
         try:
             crypto.sha3Hash(None)
         except TypeError:
@@ -42,7 +42,7 @@ class OnionrCryptoTests(unittest.TestCase):
         
         hasher.update(b'test')
         normal = hasher.hexdigest()
-        self.assertTrue(crypto.sha3Hash(b'test') == normal)
+        self.assertEqual(crypto.sha3Hash(b'test'), normal)
         
     def valid_default_id(self):
         self.assertTrue(c._utils.validatePubKey(crypto.pubKey))
@@ -73,8 +73,8 @@ class OnionrCryptoTests(unittest.TestCase):
         # Small chance that the randomized list will be same. Rerun test a couple times if it fails
         startList = ['cat', 'dog', 'moose', 'rabbit', 'monkey', 'crab', 'human', 'dolphin', 'whale', 'etc'] * 10
 
-        self.assertFalse(startList == list(crypto.randomShuffle(startList)))
+        self.assertNotEqual(startList, list(crypto.randomShuffle(startList)))
-        self.assertTrue(len(startList) == len(startList))
+        self.assertTrue(len(list(crypto.randomShuffle(startList))) == len(startList))
 
     def test_asymmetric(self):
         keyPair = crypto.generatePubKey()
@@ -127,4 +127,30 @@ class OnionrCryptoTests(unittest.TestCase):
         else:
             self.assertFalse(True)
     
+    def test_deterministic(self):
+        password = os.urandom(32)
+        gen = crypto.generateDeterministic(password)
+        self.assertTrue(c._utils.validatePubKey(gen[0]))
+        try:
+            crypto.generateDeterministic('weakpassword')
+        except onionrexceptions.PasswordStrengthError:
+            pass
+        else:
+            self.assertFalse(True)
+        try:
+            crypto.generateDeterministic(None)
+        except TypeError:
+            pass
+        else:
+            self.assertFalse(True)
+        
+        gen = crypto.generateDeterministic('weakpassword', bypassCheck=True)
+        
+        password = base64.b64encode(os.urandom(32))
+        gen1 = crypto.generateDeterministic(password)
+        gen2 = crypto.generateDeterministic(password)
+        self.assertFalse(gen == gen1)
+        self.assertTrue(gen1 == gen2)
+        self.assertTrue(c._utils.validatePubKey(gen1[0]))
+
 unittest.main()

onionr/tests/test_onionrusers.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import sys, os
 sys.path.append(".")
-import unittest, uuid, hashlib
+import unittest, uuid
 import json
 TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
 print("Test directory:", TEST_DIR)
@@ -44,7 +44,7 @@ class OnionrUserTests(unittest.TestCase):
             data = data.read()
         
         data = json.loads(data)
-        self.assertTrue(data['alias'] == 'bob')
+        self.assertEqual(data['alias'], 'bob')
     
     def test_contact_get_info(self):
         contact = c._crypto.generatePubKey()[0]
@@ -54,9 +54,16 @@ class OnionrUserTests(unittest.TestCase):
         with open(fileLocation, 'w') as contactFile:
             contactFile.write('{"alias": "bob"}')
         
-        self.assertTrue(contact.get_info('alias', forceReload=True) == 'bob')
+        self.assertEqual(contact.get_info('alias', forceReload=True), 'bob')
-        self.assertTrue(contact.get_info('fail', forceReload=True) == None)
+        self.assertEqual(contact.get_info('fail', forceReload=True), None)
-        self.assertTrue(contact.get_info('fail') == None)
+        self.assertEqual(contact.get_info('fail'), None)
+    
+    def test_encrypt(self):
+        contactPair = c._crypto.generatePubKey()
+        contact = contactmanager.ContactManager(c, contactPair[0], saveUser=True)
+        encrypted = contact.encrypt('test')
+        decrypted = c._crypto.pubKeyDecrypt(encrypted, privkey=contactPair[1], encodedData=True).decode()
+        self.assertEqual('test', decrypted)
     
     def test_delete_contact(self):
         contact = c._crypto.generatePubKey()[0]

requirements.in

@@ -0,0 +1,9 @@
+urllib3==1.23
+requests==2.20.0
+PyNaCl==1.2.1
+gevent==1.3.6
+defusedxml==0.5.0
+Flask==1.0.2
+PySocks==1.6.8
+stem==1.6.0
+deadsimplekv==0.0.1

requirements.txt

@@ -1,8 +1,196 @@
-urllib3==1.23
+#
-requests==2.20.0
+# This file is autogenerated by pip-compile
-PyNaCl==1.2.1
+# To update, run:
-gevent==1.3.6
+#
-defusedxml==0.5.0
+#    pip-compile --generate-hashes --output-file requirements.txt requirements.in
-Flask==1.0.2
+#
-PySocks==1.6.8
+certifi==2018.11.29 \
-stem==1.6.0
+    --hash=sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7 \
+    --hash=sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033 \
+    # via requests
+cffi==1.12.2 \
+    --hash=sha256:00b97afa72c233495560a0793cdc86c2571721b4271c0667addc83c417f3d90f \
+    --hash=sha256:0ba1b0c90f2124459f6966a10c03794082a2f3985cd699d7d63c4a8dae113e11 \
+    --hash=sha256:0bffb69da295a4fc3349f2ec7cbe16b8ba057b0a593a92cbe8396e535244ee9d \
+    --hash=sha256:21469a2b1082088d11ccd79dd84157ba42d940064abbfa59cf5f024c19cf4891 \
+    --hash=sha256:2e4812f7fa984bf1ab253a40f1f4391b604f7fc424a3e21f7de542a7f8f7aedf \
+    --hash=sha256:2eac2cdd07b9049dd4e68449b90d3ef1adc7c759463af5beb53a84f1db62e36c \
+    --hash=sha256:2f9089979d7456c74d21303c7851f158833d48fb265876923edcb2d0194104ed \
+    --hash=sha256:3dd13feff00bddb0bd2d650cdb7338f815c1789a91a6f68fdc00e5c5ed40329b \
+    --hash=sha256:4065c32b52f4b142f417af6f33a5024edc1336aa845b9d5a8d86071f6fcaac5a \
+    --hash=sha256:51a4ba1256e9003a3acf508e3b4f4661bebd015b8180cc31849da222426ef585 \
+    --hash=sha256:59888faac06403767c0cf8cfb3f4a777b2939b1fbd9f729299b5384f097f05ea \
+    --hash=sha256:59c87886640574d8b14910840327f5cd15954e26ed0bbd4e7cef95fa5aef218f \
+    --hash=sha256:610fc7d6db6c56a244c2701575f6851461753c60f73f2de89c79bbf1cc807f33 \
+    --hash=sha256:70aeadeecb281ea901bf4230c6222af0248c41044d6f57401a614ea59d96d145 \
+    --hash=sha256:71e1296d5e66c59cd2c0f2d72dc476d42afe02aeddc833d8e05630a0551dad7a \
+    --hash=sha256:8fc7a49b440ea752cfdf1d51a586fd08d395ff7a5d555dc69e84b1939f7ddee3 \
+    --hash=sha256:9b5c2afd2d6e3771d516045a6cfa11a8da9a60e3d128746a7fe9ab36dfe7221f \
+    --hash=sha256:9c759051ebcb244d9d55ee791259ddd158188d15adee3c152502d3b69005e6bd \
+    --hash=sha256:b4d1011fec5ec12aa7cc10c05a2f2f12dfa0adfe958e56ae38dc140614035804 \
+    --hash=sha256:b4f1d6332339ecc61275bebd1f7b674098a66fea11a00c84d1c58851e618dc0d \
+    --hash=sha256:c030cda3dc8e62b814831faa4eb93dd9a46498af8cd1d5c178c2de856972fd92 \
+    --hash=sha256:c2e1f2012e56d61390c0e668c20c4fb0ae667c44d6f6a2eeea5d7148dcd3df9f \
+    --hash=sha256:c37c77d6562074452120fc6c02ad86ec928f5710fbc435a181d69334b4de1d84 \
+    --hash=sha256:c8149780c60f8fd02752d0429246088c6c04e234b895c4a42e1ea9b4de8d27fb \
+    --hash=sha256:cbeeef1dc3c4299bd746b774f019de9e4672f7cc666c777cd5b409f0b746dac7 \
+    --hash=sha256:e113878a446c6228669144ae8a56e268c91b7f1fafae927adc4879d9849e0ea7 \
+    --hash=sha256:e21162bf941b85c0cda08224dade5def9360f53b09f9f259adb85fc7dd0e7b35 \
+    --hash=sha256:fb6934ef4744becbda3143d30c6604718871495a5e36c408431bf33d9c146889 \
+    # via pynacl
+chardet==3.0.4 \
+    --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
+    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \
+    # via requests
+click==7.0 \
+    --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
+    --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
+    # via flask
+deadsimplekv==0.0.1 \
+    --hash=sha256:1bb78e4feb01d975e89e81cac7b0141666a14ebefa06fffc1c2d86c3308e3930
+defusedxml==0.5.0 \
+    --hash=sha256:24d7f2f94f7f3cb6061acb215685e5125fbcdc40a857eff9de22518820b0a4f4 \
+    --hash=sha256:702a91ade2968a82beb0db1e0766a6a273f33d4616a6ce8cde475d8e09853b20
+flask==1.0.2 \
+    --hash=sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48 \
+    --hash=sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05
+gevent==1.3.6 \
+    --hash=sha256:03d03ea4f33e535b0a99b6be2696fde9c7417022b8ee67fb15b78f47672a0b86 \
+    --hash=sha256:13a0e74432ede9efdad5fd9aed73bd30bcfc73ddcbffe719849210f4546db833 \
+    --hash=sha256:23d623b41a431e04a9410b046520778517f5304dfbb9bfd3b1bbcc722eeaeea5 \
+    --hash=sha256:2f82d8b4d09285ca4aef34ae5c093ccf966da90e7db3bd34764ffb014c8bfa68 \
+    --hash=sha256:3223eb697d819d73dedc9a55b3dfa0cc1931e6459df4f0bf83c7c27ca256a3bd \
+    --hash=sha256:3c00ade4ae707dd6a17d6d56ebac689dc56719b83389f9aeb6a10b1e01326177 \
+    --hash=sha256:652bdd59afb330ad95550bda6864b87876e977aa4e48b9216235d932368e1987 \
+    --hash=sha256:7b413c391e8ad6607b7f7540d698a94349abd64e4935184c595f7cdcc69904c6 \
+    --hash=sha256:7feaf556fe2dc94340b603a3bfb00fbb526aaafcb8d804960939244ace4a262f \
+    --hash=sha256:810ae07c1baee83cb3d54f7dca236803554659dc00ef662ac962e4e4fd3e79bb \
+    --hash=sha256:86fa642228b8fc6a8fa268efab20440bb26599d28814e8dcd99af5dc92da10d7 \
+    --hash=sha256:a9a0a61f8dc652b3dd5dc8b9f5f9ace5d2f91f5e81f368e9ef180c8eec968234 \
+    --hash=sha256:ac3d258521b1056acb922b3aa77031a64888bb8cda1f7f6f370692cf3e224761 \
+    --hash=sha256:af7b0d16541dea42f1eceac4a02815ea3ebd8fe1eb6fc714c81ab1842ec259d4 \
+    --hash=sha256:bafef5a426473b52648c25d0ff9027aa8806982b57f8bc03abcc5f4669bfe19f \
+    --hash=sha256:bc31cdec2e584106c026a4fd24f800cb575ea8ebfcce7974b630b65d61cf36df \
+    --hash=sha256:cc42af305cb7bf1766b0084011520a81e56315dcc5b7662c209ef71a00764634 \
+    --hash=sha256:e01223b43b2e9d92733ab9953038c7a99b9c3cdb32dc865b9ce94f03a2199f96 \
+    --hash=sha256:e57f9d267b45ef9e3eb0e234307faaffa5a79cdb1477afa1befbf04de0cd8cbe \
+    --hash=sha256:e9e2942704f7fe75064ef0bc17ba46b097a57ec0e70eca1d790d5a3edb691628 \
+    --hash=sha256:f2ca6fc669def8e622b4a10809f6f6a4b6a822a1cc1175b89ad8eb34235aaa2e \
+    --hash=sha256:f456a6321f0955e802e305946ce7e7d672a7da313417ea4b4add6809630d3b0e \
+    --hash=sha256:ff8e09696a8c9100b1c88066ee44b50fbbea367ae91d830910561c902d1e7f3c
+greenlet==0.4.15 \
+    --hash=sha256:000546ad01e6389e98626c1367be58efa613fa82a1be98b0c6fc24b563acc6d0 \
+    --hash=sha256:0d48200bc50cbf498716712129eef819b1729339e34c3ae71656964dac907c28 \
+    --hash=sha256:23d12eacffa9d0f290c0fe0c4e81ba6d5f3a5b7ac3c30a5eaf0126bf4deda5c8 \
+    --hash=sha256:37c9ba82bd82eb6a23c2e5acc03055c0e45697253b2393c9a50cef76a3985304 \
+    --hash=sha256:51503524dd6f152ab4ad1fbd168fc6c30b5795e8c70be4410a64940b3abb55c0 \
+    --hash=sha256:8041e2de00e745c0e05a502d6e6db310db7faa7c979b3a5877123548a4c0b214 \
+    --hash=sha256:81fcd96a275209ef117e9ec91f75c731fa18dcfd9ffaa1c0adbdaa3616a86043 \
+    --hash=sha256:853da4f9563d982e4121fed8c92eea1a4594a2299037b3034c3c898cb8e933d6 \
+    --hash=sha256:8b4572c334593d449113f9dc8d19b93b7b271bdbe90ba7509eb178923327b625 \
+    --hash=sha256:9416443e219356e3c31f1f918a91badf2e37acf297e2fa13d24d1cc2380f8fbc \
+    --hash=sha256:9854f612e1b59ec66804931df5add3b2d5ef0067748ea29dc60f0efdcda9a638 \
+    --hash=sha256:99a26afdb82ea83a265137a398f570402aa1f2b5dfb4ac3300c026931817b163 \
+    --hash=sha256:a19bf883b3384957e4a4a13e6bd1ae3d85ae87f4beb5957e35b0be287f12f4e4 \
+    --hash=sha256:a9f145660588187ff835c55a7d2ddf6abfc570c2651c276d3d4be8a2766db490 \
+    --hash=sha256:ac57fcdcfb0b73bb3203b58a14501abb7e5ff9ea5e2edfa06bb03035f0cff248 \
+    --hash=sha256:bcb530089ff24f6458a81ac3fa699e8c00194208a724b644ecc68422e1111939 \
+    --hash=sha256:beeabe25c3b704f7d56b573f7d2ff88fc99f0138e43480cecdfcaa3b87fe4f87 \
+    --hash=sha256:d634a7ea1fc3380ff96f9e44d8d22f38418c1c381d5fac680b272d7d90883720 \
+    --hash=sha256:d97b0661e1aead761f0ded3b769044bb00ed5d33e1ec865e891a8b128bf7c656 \
+    # via gevent
+idna==2.7 \
+    --hash=sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e \
+    --hash=sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16 \
+    # via requests
+itsdangerous==1.1.0 \
+    --hash=sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19 \
+    --hash=sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749 \
+    # via flask
+jinja2==2.10 \
+    --hash=sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd \
+    --hash=sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4 \
+    # via flask
+markupsafe==1.1.1 \
+    --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \
+    --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \
+    --hash=sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235 \
+    --hash=sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5 \
+    --hash=sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff \
+    --hash=sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b \
+    --hash=sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1 \
+    --hash=sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e \
+    --hash=sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183 \
+    --hash=sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66 \
+    --hash=sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1 \
+    --hash=sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1 \
+    --hash=sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e \
+    --hash=sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b \
+    --hash=sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905 \
+    --hash=sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735 \
+    --hash=sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d \
+    --hash=sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e \
+    --hash=sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d \
+    --hash=sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c \
+    --hash=sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21 \
+    --hash=sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2 \
+    --hash=sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5 \
+    --hash=sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b \
+    --hash=sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6 \
+    --hash=sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f \
+    --hash=sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f \
+    --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 \
+    # via jinja2
+pycparser==2.19 \
+    --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \
+    # via cffi
+pynacl==1.2.1 \
+    --hash=sha256:04e30e5bdeeb2d5b34107f28cd2f5bbfdc6c616f3be88fc6f53582ff1669eeca \
+    --hash=sha256:0bfa0d94d2be6874e40f896e0a67e290749151e7de767c5aefbad1121cad7512 \
+    --hash=sha256:11aa4e141b2456ce5cecc19c130e970793fa3a2c2e6fbb8ad65b28f35aa9e6b6 \
+    --hash=sha256:13bdc1fe084ff9ac7653ae5a924cae03bf4bb07c6667c9eb5b6eb3c570220776 \
+    --hash=sha256:14339dc233e7a9dda80a3800e64e7ff89d0878ba23360eea24f1af1b13772cac \
+    --hash=sha256:1d33e775fab3f383167afb20b9927aaf4961b953d76eeb271a5703a6d756b65b \
+    --hash=sha256:2a42b2399d0428619e58dac7734838102d35f6dcdee149e0088823629bf99fbb \
+    --hash=sha256:2dce05ac8b3c37b9e2f65eab56c544885607394753e9613fd159d5e2045c2d98 \
+    --hash=sha256:63cfccdc6217edcaa48369191ae4dca0c390af3c74f23c619e954973035948cd \
+    --hash=sha256:6453b0dae593163ffc6db6f9c9c1597d35c650598e2c39c0590d1757207a1ac2 \
+    --hash=sha256:73a5a96fb5fbf2215beee2353a128d382dbca83f5341f0d3c750877a236569ef \
+    --hash=sha256:8abb4ef79161a5f58848b30ab6fb98d8c466da21fdd65558ce1d7afc02c70b5f \
+    --hash=sha256:8ac1167195b32a8755de06efd5b2d2fe76fc864517dab66aaf65662cc59e1988 \
+    --hash=sha256:8f505f42f659012794414fa57c498404e64db78f1d98dfd40e318c569f3c783b \
+    --hash=sha256:9c8a06556918ee8e3ab48c65574f318f5a0a4d31437fc135da7ee9d4f9080415 \
+    --hash=sha256:a1e25fc5650cf64f01c9e435033e53a4aca9de30eb9929d099f3bb078e18f8f2 \
+    --hash=sha256:be71cd5fce04061e1f3d39597f93619c80cdd3558a6c9ba99a546f144a8d8101 \
+    --hash=sha256:c5b1a7a680218dee9da0f1b5e24072c46b3c275d35712bc1d505b85bb03441c0 \
+    --hash=sha256:cb785db1a9468841a1265c9215c60fe5d7af2fb1b209e3316a152704607fc582 \
+    --hash=sha256:cf6877124ae6a0698404e169b3ba534542cfbc43f939d46b927d956daf0a373a \
+    --hash=sha256:d0eb5b2795b7ee2cbcfcadacbe95a13afbda048a262bd369da9904fecb568975 \
+    --hash=sha256:d3a934e2b9f20abac009d5b6951067cfb5486889cb913192b4d8288b216842f1 \
+    --hash=sha256:d795f506bcc9463efb5ebb0f65ed77921dcc9e0a50499dedd89f208445de9ecb \
+    --hash=sha256:d8aaf7e5d6b0e0ef7d6dbf7abeb75085713d0100b4eb1a4e4e857de76d77ac45 \
+    --hash=sha256:de2aaca8386cf4d70f1796352f2346f48ddb0bed61dc43a3ce773ba12e064031 \
+    --hash=sha256:e0d38fa0a75f65f556fb912f2c6790d1fa29b7dd27a1d9cc5591b281321eaaa9 \
+    --hash=sha256:eb2acabbd487a46b38540a819ef67e477a674481f84a82a7ba2234b9ba46f752 \
+    --hash=sha256:eeee629828d0eb4f6d98ac41e9a3a6461d114d1d0aa111a8931c049359298da0 \
+    --hash=sha256:f5836463a3c0cca300295b229b6c7003c415a9d11f8f9288ddbd728e2746524c \
+    --hash=sha256:f5ce9e26d25eb0b2d96f3ef0ad70e1d3ae89b5d60255c462252a3e456a48c053 \
+    --hash=sha256:fabf73d5d0286f9e078774f3435601d2735c94ce9e514ac4fb945701edead7e4
+pysocks==1.6.8 \
+    --hash=sha256:3fe52c55890a248676fd69dc9e3c4e811718b777834bcaab7a8125cf9deac672
+requests==2.20.0 \
+    --hash=sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c \
+    --hash=sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279
+six==1.12.0 \
+    --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
+    --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \
+    # via pynacl
+stem==1.6.0 \
+    --hash=sha256:d7fe1fb13ed5a94d610b5ad77e9f1b3404db0ca0586ded7a34afd323e3b849ed
+urllib3==1.23 \
+    --hash=sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf \
+    --hash=sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5
+werkzeug==0.14.1 \
+    --hash=sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c \
+    --hash=sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b \
+    # via flask

run_tests.sh

@@ -1,9 +1,10 @@
 #!/bin/bash
 cd onionr;
+rm -rf testdata;
 mkdir testdata;
 ran=0
 
-
+SECONDS=0 ; 
 close () {
    rm -rf testdata;
    exit 10;
@@ -13,5 +14,4 @@ for f in tests/*.py; do
   python3 "$f"   || close # if needed 
   let "ran++"
 done
-rm -rf testdata;
+echo "ran $ran test files successfully in $SECONDS seconds"
-echo "ran $ran test files successfully"