From 1318bb3ea6cdecfed16c40aecd7cd34327eee154 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 3 Sep 2018 10:28:08 -0500 Subject: [PATCH 1/3] use new gevent version to support python 3.7 --- onionr/api.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index d0407388..adfabf88 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -20,7 +20,7 @@ import flask from flask import request, Response, abort, send_from_directory from multiprocessing import Process -from gevent.wsgi import WSGIServer +from gevent.pywsgi import WSGIServer import sys, random, threading, hmac, hashlib, base64, time, math, os, json from core import Core from onionrblockapi import Block diff --git a/requirements.txt b/requirements.txt index 4653eaf0..97f8969e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ urllib3==1.23 requests==2.18.4 PyNaCl==1.2.1 -gevent==1.2.2 +gevent==1.3.6 sha3==0.2.1 defusedxml==0.5.0 simple_crypt==4.1.7 From 851b2bef76650491db8f82eb72889e31ee270fbd Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 13 Nov 2018 11:07:46 -0600 Subject: [PATCH 2/3] onionr now waits for a block to be uploaded to share it, unless the node exits --- onionr/api.py | 17 +++++++++++++- onionr/communicator2.py | 50 ++++++++++++++++++++++++----------------- onionr/core.py | 2 ++ onionr/onionr.py | 2 +- onionr/onionrutils.py | 4 +++- 5 files changed, 51 insertions(+), 24 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index 3efc11b8..f4b4cbe9 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -105,6 +105,8 @@ class API: self.mimeType = 'text/plain' self.overrideCSP = False + self.hideBlocks = [] # Blocks to be denied sharing + with open(self._core.dataDir + 'time-bypass.txt', 'w') as bypass: bypass.write(self.timeBypassToken) @@ -236,6 +238,15 @@ class API: self.validateHost('private') if action == 'hello': resp = Response('Hello, World! ' + request.host) + elif action == 'waitForShare': + if self._core._utils.validateHash(data): + if data not in self.hideBlocks: + self.hideBlocks.append(data) + else: + self.hideBlocks.remove(data) + resp = "success" + else: + resp = "failed to validate hash" elif action == 'shutdown': # request.environ.get('werkzeug.server.shutdown')() self.http_server.stop() @@ -464,7 +475,11 @@ class API: elif action == 'getDBHash': resp = Response(self._utils.getBlockDBHash()) elif action == 'getBlockHashes': - resp = Response('\n'.join(self._core.getBlockList())) + bList = self._core.getBlockList() + for b in self.hideBlocks: + if b in bList: + bList.remove(b) + resp = Response('\n'.join(bList)) # setData should be something the communicator initiates, not this api elif action == 'getData': resp = '' diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 08806569..53c35cb6 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -41,7 +41,7 @@ class OnionrCommunicatorDaemon: self.nistSaltTimestamp = 0 self.powSalt = 0 - self.blockToUpload = '' + self.blocksToUpload = [] # loop time.sleep delay in seconds self.delay = 1 @@ -96,6 +96,7 @@ class OnionrCommunicatorDaemon: OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65) OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True) + OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True, maxThreads=1) cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True) @@ -167,7 +168,7 @@ class OnionrCommunicatorDaemon: else: continue newDBHash = self.peerAction(peer, 'getDBHash') # get their db hash - if newDBHash == False: + if newDBHash == False or not self._core._utils.validateHash(newDBHash): continue # if request failed, restart loop (peer is added to offline peers automatically) triedPeers.append(peer) if newDBHash != self._core.getAddressInfo(peer, 'DBHash'): @@ -466,8 +467,7 @@ class OnionrCommunicatorDaemon: if i.timerFunction.__name__ == 'lookupAdders': i.count = (i.frequency - 1) elif cmd[0] == 'uploadBlock': - self.blockToUpload = cmd[1] - threading.Thread(target=self.uploadBlock).start() + self.blocksToUpload.append(cmd[1]) elif cmd[0] == 'startSocket': # Create our own socket server socketInfo = json.loads(cmd[1]) @@ -488,23 +488,31 @@ class OnionrCommunicatorDaemon: '''Upload our block to a few peers''' # when inserting a block, we try to upload it to a few peers to add some deniability triedPeers = [] - if not self._core._utils.validateHash(self.blockToUpload): - logger.warn('Requested to upload invalid block') - return - for i in range(max(len(self.onlinePeers), 2)): - peer = self.pickOnlinePeer() - if peer in triedPeers: - continue - triedPeers.append(peer) - url = 'http://' + peer + '/public/upload/' - data = {'block': block.Block(self.blockToUpload).getRaw()} - proxyType = '' - if peer.endswith('.onion'): - proxyType = 'tor' - elif peer.endswith('.i2p'): - proxyType = 'i2p' - logger.info("Uploading block") - self._core._utils.doPostRequest(url, data=data, proxyType=proxyType) + finishedUploads = [] + if len(self.blocksToUpload) != 0: + for bl in self.blocksToUpload: + if not self._core._utils.validateHash(bl): + logger.warn('Requested to upload invalid block') + return + for i in range(max(len(self.onlinePeers), 2)): + peer = self.pickOnlinePeer() + if peer in triedPeers: + continue + triedPeers.append(peer) + url = 'http://' + peer + '/public/upload/' + data = {'block': block.Block(bl).getRaw()} + proxyType = '' + if peer.endswith('.onion'): + proxyType = 'tor' + elif peer.endswith('.i2p'): + proxyType = 'i2p' + logger.info("Uploading block") + if not self._core._utils.doPostRequest(url, data=data, proxyType=proxyType) == False: + self._core._utils.localCommand('waitForShare', data=bl) + finishedUploads.append(bl) + for x in finishedUploads: + self.blocksToUpload.remove(x) + self.decrementThreadCount('uploadBlock') def announce(self, peer): '''Announce to peers our address''' diff --git a/onionr/core.py b/onionr/core.py index fab7b04a..cb209061 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -728,6 +728,8 @@ class Core: payload = proof.waitForResult() if payload != False: retData = self.setData(payload) + # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult + self._utils.localCommand('waitForShare', data=retData) self.addToBlockDB(retData, selfInsert=True, dataSaved=True) #self.setBlockType(retData, meta['type']) self._utils.processBlockMetadata(retData) diff --git a/onionr/onionr.py b/onionr/onionr.py index eceb7639..e886d4d1 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -40,7 +40,7 @@ except ImportError: raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)") ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech' -ONIONR_VERSION = '0.3.0' # for debugging and stuff +ONIONR_VERSION = '0.3.1' # for debugging and stuff ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION) API_VERSION = '5' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you. diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 558a6b20..936d1d47 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -150,7 +150,7 @@ class OnionrUtils: logger.error('Failed to read my address.', error = error) return None - def localCommand(self, command, silent = True): + def localCommand(self, command, data='', silent = True): ''' Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers. ''' @@ -164,6 +164,8 @@ class OnionrUtils: except FileNotFoundError: return False payload = 'http://%s:%s/client/?action=%s&token=%s&timingToken=%s' % (hostname, config.get('client.port'), command, config.get('client.hmac'), self.timingToken) + if data != '': + payload += '&data=' + data try: retData = requests.get(payload).text except Exception as error: From 1298e7cd7dea0a79e6f5b03830ecf98cb483d7ee Mon Sep 17 00:00:00 2001 From: Aaron <20esaua@ttsdstudents.org> Date: Tue, 13 Nov 2018 21:04:14 +0000 Subject: [PATCH 3/3] urlencode data variable --- onionr/onionrutils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 936d1d47..a4cb6263 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -18,7 +18,7 @@ along with this program. If not, see . ''' # Misc functions that do not fit in the main api, but are useful -import getpass, sys, requests, os, socket, hashlib, logger, sqlite3, config, binascii, time, base64, json, glob, shutil, math, json, re +import getpass, sys, requests, os, socket, hashlib, logger, sqlite3, config, binascii, time, base64, json, glob, shutil, math, json, re, urllib.parse import nacl.signing, nacl.encoding from onionrblockapi import Block import onionrexceptions @@ -165,7 +165,7 @@ class OnionrUtils: return False payload = 'http://%s:%s/client/?action=%s&token=%s&timingToken=%s' % (hostname, config.get('client.port'), command, config.get('client.hmac'), self.timingToken) if data != '': - payload += '&data=' + data + payload += '&data=' + urllib.parse.quote_plus(data) try: retData = requests.get(payload).text except Exception as error: