From 7706d4e04c828b03c5c185b3d190f3f8c1021cef Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 18:35:00 -0500 Subject: [PATCH 01/15] added self check, bootstrap node file --- onionr/communicator.py | 9 ++++++++- onionr/core.py | 19 +++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/onionr/communicator.py b/onionr/communicator.py index e8a27b24..da36179b 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -263,13 +263,20 @@ class OnionrCommunicate: ''' return urllib.parse.quote_plus(data) - def performGet(self, action, peer, data=None, skipHighFailureAddress=False, peerType='tor'): + def performGet(self, action, peer, data=None, skipHighFailureAddress=False, peerType='tor', selfCheck=True): ''' Performs a request to a peer through Tor or i2p (currently only Tor) ''' if not peer.endswith('.onion') and not peer.endswith('.onion/'): raise PeerError('Currently only Tor .onion peers are supported. You must manually specify .onion') + + if len(self._core.hsAdder.strip()) == 0: + raise Exception("Could not perform self address check in performGet due to not knowing our address") + if selfCheck: + if peer.replace('/', '') == self._core.hsAdder: + logger.warn('Tried to performget to own hidden service, but selfCheck was not set to false') + return # Store peer in peerData dictionary (non permanent) if not peer in self.peerData: diff --git a/onionr/core.py b/onionr/core.py index f363d8bc..8fb5fd82 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -44,6 +44,9 @@ class Core: self.addressDB = 'data/address.db' self.hsAdder = '' + self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt' + self.bootstrapList = [] + if not os.path.exists('data/'): os.mkdir('data/') if not os.path.exists('data/blocks/'): @@ -54,10 +57,20 @@ class Core: if os.path.exists('data/hs/hostname'): with open('data/hs/hostname', 'r') as hs: self.hsAdder = hs.read() + else: + logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation) + + # Load bootstrap address list + if os.path.exists(self.bootstrapFileLocation): + with open(self.bootstrapFileLocation, 'r') as bootstrap: + bootstrap = bootstrap.read() + for i in bootstrap.split('\n'): + self.bootstrapList.append(i) self._utils = onionrutils.OnionrUtils(self) # Initialize the crypto object self._crypto = onionrcrypto.OnionrCrypto(self) + except Exception as error: logger.error('Failed to initialize core Onionr library.', error=error) logger.fatal('Cannot recover from error.') @@ -574,8 +587,10 @@ class Core: announceAmount = 2 nodeList = self.listAdders() if len(nodeList) == 0: - self.addAddress('onionragxuddecmg.onion') - nodeList.append('onionragxuddecmg.onion') + for i in self.bootstrapList: + if self._utils.validateID(i): + self.addAddress(i) + nodeList.append(i) if announceAmount > len(nodeList): announceAmount = len(nodeList) for i in range(announceAmount): From c9cfeac30334f41d9fe611984498643fd6a2d755 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 18:36:09 -0500 Subject: [PATCH 02/15] added self check, bootstrap node file --- onionr/static-data/bootstrap-nodes.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 onionr/static-data/bootstrap-nodes.txt diff --git a/onionr/static-data/bootstrap-nodes.txt b/onionr/static-data/bootstrap-nodes.txt new file mode 100644 index 00000000..7afadcc1 --- /dev/null +++ b/onionr/static-data/bootstrap-nodes.txt @@ -0,0 +1 @@ +onionisrgccylxpr.onion From 4ff6baa279256deaa19ecd4eaba0dcf9e1894cfa Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 18:38:32 -0500 Subject: [PATCH 03/15] fixed messed up message for missing bootstrap file --- onionr/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index 8fb5fd82..78bd6aff 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -57,8 +57,6 @@ class Core: if os.path.exists('data/hs/hostname'): with open('data/hs/hostname', 'r') as hs: self.hsAdder = hs.read() - else: - logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation) # Load bootstrap address list if os.path.exists(self.bootstrapFileLocation): @@ -66,6 +64,8 @@ class Core: bootstrap = bootstrap.read() for i in bootstrap.split('\n'): self.bootstrapList.append(i) + else: + logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation) self._utils = onionrutils.OnionrUtils(self) # Initialize the crypto object From 89f1b11dac3d06918a3fd67c6255147b1a55e6b5 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 20:43:17 -0500 Subject: [PATCH 04/15] started work on base64 transporting --- onionr/api.py | 33 +++++++++++++++++++++----- onionr/communicator.py | 6 ++++- onionr/core.py | 6 ++--- onionr/onionr.py | 19 ++++++++++++++- onionr/static-data/bootstrap-nodes.txt | 1 + 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index b43908e1..0f83a4de 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -68,6 +68,8 @@ class API: self.clientToken = config.get('client')['client_hmac'] self.timeBypassToken = base64.b16encode(os.urandom(32)).decode() + self.mimeType = 'text/plain' + with open('data/time-bypass.txt', 'w') as bypass: bypass.write(self.timeBypassToken) @@ -96,12 +98,17 @@ class API: def afterReq(resp): if not self.requestFailed: resp.headers['Access-Control-Allow-Origin'] = '*' - else: - resp.headers['server'] = 'Onionr' - resp.headers['Content-Type'] = 'text/plain' - resp.headers["Content-Security-Policy"] = "default-src 'none'" + #else: + # resp.headers['server'] = 'Onionr' + resp.headers['Content-Type'] = self.mimeType + resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" resp.headers['X-Frame-Options'] = 'deny' resp.headers['X-Content-Type-Options'] = "nosniff" + resp.headers['server'] = 'Onionr' + + # reset to text/plain to help prevent browser attacks + if self.mimeType != 'text/plain': + self.mimeType = 'text/plain' return resp @@ -111,6 +118,11 @@ class API: timingToken = '' else: timingToken = request.args.get('timingToken') + data = request.args.get('data') + try: + data = data + except: + data = '' startTime = math.floor(time.time()) # we should keep a hash DB of requests (with hmac) to prevent replays action = request.args.get('action') @@ -129,6 +141,15 @@ class API: resp = Response('pong') elif action == 'stats': resp = Response('me_irl') + elif action == 'site': + block = data + siteData = self._core.getData(data) + response = 'not found' + if siteData != '' and siteData != False: + self.mimeType = 'text/html' + response = siteData.split('-', 2)[-1] + resp = Response(response) + else: resp = Response('(O_o) Dude what? (invalid command)') endTime = math.floor(time.time()) @@ -149,7 +170,7 @@ class API: requestingPeer = request.args.get('myID') data = request.args.get('data') try: - data + data = data except: data = '' if action == 'firstConnect': @@ -175,7 +196,7 @@ class API: resp = Response('') # setData should be something the communicator initiates, not this api elif action == 'getData': - resp = self._core.getData(data) + resp = base64.b64encode(self._core.getData(data)) if resp == False: abort(404) resp = "" diff --git a/onionr/communicator.py b/onionr/communicator.py index da36179b..f316d14f 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -19,7 +19,7 @@ and code to operate as a daemon, getting commands from the command queue databas You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import sqlite3, requests, hmac, hashlib, time, sys, os, math, logger, urllib.parse, random +import sqlite3, requests, hmac, hashlib, time, sys, os, math, logger, urllib.parse, random, base64 import core, onionrutils, onionrcrypto, netcontroller, onionrproofs, btc, config, onionrplugins as plugins class OnionrCommunicate: @@ -239,6 +239,10 @@ class OnionrCommunicate: for i in peerList: hasher = hashlib.sha3_256() data = self.performGet('getData', i, hash) + try: + base64.b64decode(data) + except binascii.Error: + data = b'' if data == False or len(data) > 10000000: continue hasher.update(data.encode()) diff --git a/onionr/core.py b/onionr/core.py index 78bd6aff..4b6d8d88 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -258,7 +258,7 @@ class Core: Simply return the data associated to a hash ''' try: - dataFile = open(self.blockDataLocation + hash + '.dat') + dataFile = open(self.blockDataLocation + hash + '.dat', 'rb') data = dataFile.read() dataFile.close() except FileNotFoundError: @@ -281,8 +281,8 @@ class Core: pass # TODO: properly check if block is already saved elsewhere #raise Exception("Data is already set for " + dataHash) else: - blockFile = open(blockFileName, 'w') - blockFile.write(data.decode()) + blockFile = open(blockFileName, 'wb') + blockFile.write(data) blockFile.close() conn = sqlite3.connect(self.blockDB) diff --git a/onionr/onionr.py b/onionr/onionr.py index 395dcb75..e9bd8f15 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -175,6 +175,7 @@ class Onionr: 'add-addr': self.addAddress, 'addaddr': self.addAddress, 'addaddress': self.addAddress, + 'addfile': self.addFile, 'introduce': self.onionrCore.introduceNode, 'connect': self.addAddress @@ -196,6 +197,7 @@ class Onionr: 'add-msg': 'Broadcasts a message to the Onionr network', 'pm': 'Adds a private message to block', 'get-pms': 'Shows private messages sent to you', + 'addfile': 'Create an Onionr block from a file', 'introduce': 'Introduce your node to the public Onionr network (DAEMON MUST BE RUNNING)', } @@ -369,7 +371,7 @@ class Onionr: addedHash = self.onionrCore.setData(messageToAdd) self.onionrCore.addToBlockDB(addedHash, selfInsert=True) self.onionrCore.setBlockType(addedHash, 'txt') - + logger.info("inserted your message as block: " + addedHash) return def getPMs(self): @@ -556,5 +558,20 @@ class Onionr: retval = retVal.read() except FileNotFoundError: return retVal + + def addFile(self): + '''command to add a file to the onionr network''' + if len(sys.argv) >= 2: + newFile = sys.argv[2] + logger.info('Attempting to add file...') + try: + with open(newFile, 'r') as new: + new = new.read() + except FileNotFoundError: + logger.warn('That file does not exist. Improper path?') + else: + print(new) + self.onionrCore.insertBlock(new, header='bin') + Onionr() diff --git a/onionr/static-data/bootstrap-nodes.txt b/onionr/static-data/bootstrap-nodes.txt index 7afadcc1..72e08794 100644 --- a/onionr/static-data/bootstrap-nodes.txt +++ b/onionr/static-data/bootstrap-nodes.txt @@ -1 +1,2 @@ onionisrgccylxpr.onion +aaronk3mcmglj6qedwptg62yl3wxxjwba2ucpoobrn7iudcacdxtrfad.onion From 23d092fb94d5931973885945ac4ed71458d23f5c Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 20:46:22 -0500 Subject: [PATCH 05/15] added binascii import --- onionr/communicator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/communicator.py b/onionr/communicator.py index f316d14f..91e85fe4 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -19,7 +19,7 @@ and code to operate as a daemon, getting commands from the command queue databas You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import sqlite3, requests, hmac, hashlib, time, sys, os, math, logger, urllib.parse, random, base64 +import sqlite3, requests, hmac, hashlib, time, sys, os, math, logger, urllib.parse, random, base64, binascii import core, onionrutils, onionrcrypto, netcontroller, onionrproofs, btc, config, onionrplugins as plugins class OnionrCommunicate: From c562fabbbd7e2f63c254906554ad8bd9c6164f47 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 20:56:20 -0500 Subject: [PATCH 06/15] removed bytes conversion on downloadblock data --- onionr/communicator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/communicator.py b/onionr/communicator.py index 91e85fe4..19948c06 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -245,7 +245,7 @@ class OnionrCommunicate: data = b'' if data == False or len(data) > 10000000: continue - hasher.update(data.encode()) + hasher.update(data) digest = hasher.hexdigest() if type(digest) is bytes: digest = digest.decode() From 7199b0e5da36ce50d6628af59da4643591ebaf5c Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 20:59:44 -0500 Subject: [PATCH 07/15] now check base64 after bool check --- onionr/communicator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onionr/communicator.py b/onionr/communicator.py index 19948c06..cba930ef 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -239,12 +239,12 @@ class OnionrCommunicate: for i in peerList: hasher = hashlib.sha3_256() data = self.performGet('getData', i, hash) + if data == False or len(data) > 10000000: + continue try: base64.b64decode(data) except binascii.Error: data = b'' - if data == False or len(data) > 10000000: - continue hasher.update(data) digest = hasher.hexdigest() if type(digest) is bytes: From d41f86a27a1939b5896dcaaf0550779bde20a7bd Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 21:10:09 -0500 Subject: [PATCH 08/15] what --- onionr/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/onionr/api.py b/onionr/api.py index 0f83a4de..1603f995 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -196,7 +196,9 @@ class API: resp = Response('') # setData should be something the communicator initiates, not this api elif action == 'getData': - resp = base64.b64encode(self._core.getData(data)) + if self._utils.validateHash(data): + if not os.path.exists('data/blocks/' + data + '.db'): + resp = base64.b64encode(self._core.getData(data)) if resp == False: abort(404) resp = "" From ae79b7ee3a6264441682c4bc782b47882bdd54cd Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 21:13:55 -0500 Subject: [PATCH 09/15] fixed stupid mistake --- onionr/communicator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/communicator.py b/onionr/communicator.py index cba930ef..35d8bcfd 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -242,7 +242,7 @@ class OnionrCommunicate: if data == False or len(data) > 10000000: continue try: - base64.b64decode(data) + data = base64.b64decode(data) except binascii.Error: data = b'' hasher.update(data) From 885392628745f64783e42136d1fb22276c455e14 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 21:16:11 -0500 Subject: [PATCH 10/15] bumped api version and removed encode on setdata since it is always bytes now --- onionr/core.py | 2 +- onionr/onionr.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onionr/core.py b/onionr/core.py index 4b6d8d88..017c7562 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -270,7 +270,7 @@ class Core: ''' Set the data assciated with a hash ''' - data = data.encode() + data = data hasher = hashlib.sha3_256() hasher.update(data) dataHash = hasher.hexdigest() diff --git a/onionr/onionr.py b/onionr/onionr.py index e9bd8f15..ae113ba4 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -34,7 +34,7 @@ except ImportError: ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech' ONIONR_VERSION = '0.0.0' # for debugging and stuff -API_VERSION = '1' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes knows how to communicate without learning too much information about you. +API_VERSION = '2' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes knows how to communicate without learning too much information about you. class Onionr: def __init__(self): From 7578ac874b029339e63f18b9bc23a1d760d7ee51 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 21:19:57 -0500 Subject: [PATCH 11/15] now only show block contents debug if txt --- onionr/communicator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/onionr/communicator.py b/onionr/communicator.py index 35d8bcfd..ce83199f 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -251,11 +251,11 @@ class OnionrCommunicate: digest = digest.decode() if digest == hash.strip(): self._core.setData(data) - if data.startswith('-txt-'): - self._core.setBlockType(hash, 'txt') logger.info('Successfully obtained data for ' + hash, timestamp=True) - if len(data) < 120: - logger.debug('Block text:\n' + data) + if data.startswith(b'-txt-'): + self._core.setBlockType(hash, 'txt') + if len(data) < 120: + logger.debug('Block text:\n' + data) else: logger.warn("Failed to validate " + hash + " " + " hash calculated was " + digest) From c807056ef2d7aee68044b7742157c2402bd3483a Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 21:24:34 -0500 Subject: [PATCH 12/15] convert data to bytes if it isnt --- onionr/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onionr/core.py b/onionr/core.py index 017c7562..a1348279 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -272,6 +272,8 @@ class Core: ''' data = data hasher = hashlib.sha3_256() + if not type(data) is bytes: + data = data.encode() hasher.update(data) dataHash = hasher.hexdigest() if type(dataHash) is bytes: From 583480629ad330f7e6fabbe6efc8f5c5570987de Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 21:58:24 -0500 Subject: [PATCH 13/15] hopefully working again, hopefully less block validation issues --- onionr/api.py | 2 +- onionr/communicator.py | 3 ++- onionr/onionrcrypto.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/onionr/api.py b/onionr/api.py index 1603f995..9acdf0e2 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -147,7 +147,7 @@ class API: response = 'not found' if siteData != '' and siteData != False: self.mimeType = 'text/html' - response = siteData.split('-', 2)[-1] + response = siteData.split(b'-', 2)[-1] resp = Response(response) else: diff --git a/onionr/communicator.py b/onionr/communicator.py index ce83199f..2e067979 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -239,6 +239,7 @@ class OnionrCommunicate: for i in peerList: hasher = hashlib.sha3_256() data = self.performGet('getData', i, hash) + print('b64 data', data) if data == False or len(data) > 10000000: continue try: @@ -255,7 +256,7 @@ class OnionrCommunicate: if data.startswith(b'-txt-'): self._core.setBlockType(hash, 'txt') if len(data) < 120: - logger.debug('Block text:\n' + data) + logger.debug('Block text:\n' + data.decode()) else: logger.warn("Failed to validate " + hash + " " + " hash calculated was " + digest) diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 98bf9230..00e6d520 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -167,7 +167,7 @@ class OnionrCrypto: return binascii.hexlify(nacl.utils.random(nacl.secret.SecretBox.KEY_SIZE)) def generatePubKey(self): - '''Generate a Ed25519 public key pair, return tuple of base64encoded pubkey, privkey''' + '''Generate a Ed25519 public key pair, return tuple of base32encoded pubkey, privkey''' private_key = nacl.signing.SigningKey.generate() public_key = private_key.verify_key.encode(encoder=nacl.encoding.Base32Encoder()) return (public_key.decode(), private_key.encode(encoder=nacl.encoding.Base32Encoder()).decode()) \ No newline at end of file From 62737ea2c1a1ab25a6b5ef9034e7bd72dc521d48 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 22:15:20 -0500 Subject: [PATCH 14/15] added bootstrap node and removed debug message --- onionr/communicator.py | 1 - onionr/static-data/bootstrap-nodes.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/communicator.py b/onionr/communicator.py index 2e067979..a99664bf 100755 --- a/onionr/communicator.py +++ b/onionr/communicator.py @@ -239,7 +239,6 @@ class OnionrCommunicate: for i in peerList: hasher = hashlib.sha3_256() data = self.performGet('getData', i, hash) - print('b64 data', data) if data == False or len(data) > 10000000: continue try: diff --git a/onionr/static-data/bootstrap-nodes.txt b/onionr/static-data/bootstrap-nodes.txt index 72e08794..64215eb7 100644 --- a/onionr/static-data/bootstrap-nodes.txt +++ b/onionr/static-data/bootstrap-nodes.txt @@ -1,2 +1,3 @@ onionisrgccylxpr.onion aaronk3mcmglj6qedwptg62yl3wxxjwba2ucpoobrn7iudcacdxtrfad.onion +onionragxuddecmg.onion From 8300cd1c91c39fa7f573737a3c4bde033b5dbc32 Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sun, 22 Apr 2018 22:20:52 -0500 Subject: [PATCH 15/15] fixed messed up gethostname() --- onionr/onionr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onionr/onionr.py b/onionr/onionr.py index ae113ba4..bf66f736 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -555,7 +555,7 @@ class Onionr: retVal = '' try: with open('./data/hs/hostname', 'r') as hostname: - retval = retVal.read() + retVal = hostname.read() except FileNotFoundError: return retVal