diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..4a0c253e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +onionr/data/**/* +onionr/data +RUN-WINDOWS.bat +MY-RUN.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..90f16342 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM ubuntu:xenial + +#Base settings +ENV HOME /root + +#Install needed packages +RUN apt update && apt install -y python3 python3-dev python3-pip tor locales + +RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ + locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +WORKDIR /srv/ +ADD ./requirements.txt /srv/requirements.txt +RUN pip3 install -r requirements.txt + +WORKDIR /root/ +#Add Onionr source +COPY . /root/ +VOLUME /root/data/ + +#Set upstart command +CMD bash + +#Expose ports +EXPOSE 8080 diff --git a/Makefile b/Makefile index 472ffc2d..13d9c0f9 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,21 @@ +PREFIX = /usr/local + .DEFAULT_GOAL := setup setup: sudo pip3 install -r requirements.txt + -@cd onionr/static-data/ui/; ./compile.py install: - sudo rm -rf /usr/share/onionr/ - sudo rm -f /usr/bin/onionr - sudo cp -rp ./onionr /usr/share/onionr - sudo sh -c "echo \"#!/bin/sh\ncd /usr/share/onionr/\n./onionr.py \\\"\\\$$@\\\"\" > /usr/bin/onionr" - sudo chmod +x /usr/bin/onionr - sudo chown -R `whoami` /usr/share/onionr/ + cp -rfp ./onionr $(DESTDIR)$(PREFIX)/share/onionr + echo '#!/bin/sh' > $(DESTDIR)$(PREFIX)/bin/onionr + echo 'cd $(DESTDIR)$(PREFIX)/share/onionr' > $(DESTDIR)$(PREFIX)/bin/onionr + echo './onionr "$$@"' > $(DESTDIR)$(PREFIX)/bin/onionr + chmod +x $(DESTDIR)$(PREFIX)/bin/onionr uninstall: - sudo rm -rf /usr/share/onionr - sudo rm -f /usr/bin/onionr + rm -rf $(DESTDIR)$(PREFIX)/share/onionr + rm -f $(DESTDIR)$(PREFIX)/bin/onionr test: @./RUN-LINUX.sh stop @@ -26,7 +28,7 @@ test: soft-reset: @echo "Soft-resetting Onionr..." - rm -f onionr/data/blocks/*.dat onionr/data/*.db | true > /dev/null 2>&1 + rm -f onionr/data/blocks/*.dat onionr/data/*.db onionr/data/block-nonces.dat | true > /dev/null 2>&1 @./RUN-LINUX.sh version | grep -v "Failed" --color=always reset: diff --git a/docs/api.md b/docs/api.md index 7f9128a5..52a55368 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,34 +1,2 @@ -BLOCK HEADERS (simple ID system to identify block type) ------------------------------------------------ --crypt- (encrypted block) --bin- (binary file) --txt- (plaintext) - HTTP API ------------------------------------------------- -/client/ (Private info, not publicly accessible) - -- hello - - hello world -- shutdown - - exit onionr -- stats - - show node stats - -/public/ - -- firstConnect - - initialize with peer -- ping - - pong -- setHMAC - - set a created symmetric key -- getDBHash - - get the hash of the current hash database state -- getPGP - - export node's PGP public key -- getData - - get a data block -- getBlockHashes - - get a list of the node's hashes -------------------------------------------------- +TODO diff --git a/docs/onionr-draft.md b/docs/onionr-draft.md deleted file mode 100644 index acce39e7..00000000 --- a/docs/onionr-draft.md +++ /dev/null @@ -1,57 +0,0 @@ -# Onionr Protocol Spec v2 - -A P2P platform for Tor & I2P - -# Overview - -Onionr is an encrypted microblogging & mailing system designed in the spirit of Twitter. -There are no central servers and all traffic is peer to peer by default (routed via Tor or I2P). -User IDs are simply Tor onion service/I2P host id + Ed25519 key fingerprint. -Private blocks are only able to be read by the intended peer. -All traffic is over Tor/I2P, connecting only to Tor onion and I2P hidden services. - -## Goals: - • Selective sharing of information - • Secure & semi-anonymous direct messaging - • Forward secrecy - • Defense in depth - • Data should be secure for years to come - • Decentralization - * Avoid browser-based exploits that plague similar software - * Avoid timing attacks & unexpected metadata leaks - -## Protocol - -Onionr nodes use HTTP (over Tor/I2P) to exchange keys, metadata, and blocks. Blocks are identified by their sha3_256 hash. Nodes sync a table of blocks hashes and attempt to download blocks they do not yet have from random peers. - -Blocks may be encrypted using Curve25519 or Salsa20. - -Blocks have IDs in the following format: - --Optional hash of public key of publisher (base64)-optional signature (non-optional if publisher is specified) (Base64)-block type-block hash(sha3-256) - -pubkeyHash-signature-type-hash - -## Connections - -When a node first comes online, it attempts to bootstrap using a default list provided by a client. -When two peers connect, they exchange Ed25519 keys (if applicable) then Salsa20 keys. - -Salsa20 keys are regenerated either every X many communications with a peer or every X minutes. - -Every 100kb or every 2 hours is a recommended default. - -All valid requests with HMAC should be recorded until used HMAC's expiry to prevent replay attacks. -Peer Types - * Friends: - * Encrypted ‘friends only’ posts to one another - * Usually less strict rate & storage limits - * Strangers: - * Used for storage of encrypted or public information - * Can only read public posts - * Usually stricter rate & storage limits - -## Spam mitigation - -To send or receive data, a node can optionally request that the other node generate a hash that when in hexadecimal representation contains a random string at a random location in the string. Clients will configure what difficulty to request, and what difficulty is acceptable for themselves to perform. Difficulty should correlate with recent network & disk usage and data size. Friends can be configured to have less strict (to non existent) limits, separately from strangers. (proof of work). -Rate limits can be strict, as Onionr is not intended to be an instant messaging application. \ No newline at end of file diff --git a/docs/onionr-logo.png b/docs/onionr-logo.png index d0860f6d..b6c3c9b5 100644 Binary files a/docs/onionr-logo.png and b/docs/onionr-logo.png differ diff --git a/docs/onionr-web.png b/docs/onionr-web.png new file mode 100644 index 00000000..3124253d Binary files /dev/null and b/docs/onionr-web.png differ diff --git a/docs/whitepaper.md b/docs/whitepaper.md new file mode 100644 index 00000000..e791b83c --- /dev/null +++ b/docs/whitepaper.md @@ -0,0 +1,97 @@ +

+ <h1>Onionr</h1> +

+

Anonymous, Decentralized, Distributed Network

+ +# Introduction + +The most important thing in the modern world is information. The ability to communicate freely with others. The internet has provided humanity with the ability to spread information globally, but there are many people who try (and sometimes succeed) to stifle the flow of information. + +Internet censorship comes in many forms, state censorship, corporate consolidation of media, threats of violence, network exploitation (e.g. denial of service attacks). + +To prevent censorship or loss of information, these measures must be in place: + +* Resistance to censorship of underlying infrastructure or of network hosts + +* Anonymization of users by default + * The Inability to violently coerce human users (personal threats/"doxxing", or totalitarian regime censorship) + +* Economic availability. A system should not rely on a single device to be constantly online, and should not be overly expensive to use. The majority of people in the world own cell phones, but comparatively few own personal computers, particularly in developing countries. + +There are many great projects that tackle decentralization and privacy issues, but there are none which tackle all of the above issues. Some of the existing networks have also not worked well in practice, or are more complicated than they need to be. + +# Onionr Design Goals + +When designing Onionr we had these goals in mind: + +* Anonymous Blocks + + * Difficult to determine block creator or users regardless of transport used +* Default Anonymous Transport Layer + * Tor and I2P +* Transport agnosticism +* Default global sync, but can configure what blocks to seed +* Spam resistance +* Encrypted blocks + +# Onionr Design + +(See the spec for specific details) + +## General Overview + +At its core, Onionr is merely a description for storing data in self-verifying packages ("blocks"). These blocks can be encrypted to a user (or self), encrypted symmetrically, or not at all. Blocks can be signed by their creator, but regardless, they are self-verifying due to being identified by a sha3-256 hash value; once a block is created, it cannot be modified. + +Onionr exchanges a list of blocks between all nodes. By default, all nodes download and share all other blocks, however this is configurable. + +## User IDs + +User IDs are simply Ed25519 public keys. They are represented in Base32 format, or encoded using the [PGP Word List](https://en.wikipedia.org/wiki/PGP_word_list). + +Public keys can be generated deterministicly with a password using a key derivation function (Argon2id). This password can be shared between many users in order to share data anonymously among a group, using only 1 password. This is useful in some cases, but is risky, as if one user causes the key to be compromised and does not notify the group or revoke the key, there is no way to know. + +## Nodes + +Although Onionr is transport agnostic, the only supported transports in the reference implemetation are Tor .onion services and I2P hidden services. Nodes announce their address on creation. + +### Node Profiling + +To mitigate maliciously slow or unreliable nodes, Onionr builds a profile on nodes it connects to. Nodes are assigned a score, which raises based on the amount of successful block transfers, speed, and reliabilty of a node, and reduces based on how unreliable a node is. If a node is unreachable for over 24 hours after contact, it is forgotten. Onionr can also prioritize connection to 'friend' nodes. + +## Block Format + +Onionr blocks are very simple. They are structured in two main parts: a metadata section and a data section, with a line feed delimiting where metadata ends and data begins. + +Metadata defines what kind of data is in a block, signature data, encryption settings, and other arbitrary information. + +Optionally, a random token can be inserted into the metadata for use in Proof of Work. + +### Block Encryption + +For encryption, Onionr uses ephemeral Curve25519 keys for key exchange and XSalsa20-Poly1305 as a symmetric cipher, or optionally using only XSalsa20-Poly1305 with a pre-shared key. + +Regardless of encryption, blocks can be signed internally using Ed25519. + +## Block Exchange + +Blocks can be exchanged using any method, as they are not reliant on any other blocks. + +By default, every node shares a list of the blocks it is sharing, and will download any blocks it does not yet have. + +## Spam mitigation and block storage time + +By default, an Onionr node adjusts the target difficulty for blocks to be accepted based on the percent of disk usage allocated to Onionr. + +Blocks are stored indefinitely until the allocated space is filled, at which point Onionr will remove the oldest blocks as needed, save for "pinned" blocks, which are permanently stored. + +## Block Timestamping + +Onionr can provide evidence of when a block was inserted by requesting other users to sign a hash of the current time with the block data hash: sha3_256(time + sha3_256(block data)). + +This can be done either by the creator of the block prior to generation, or by any node after insertion. + +In addition, randomness beacons such as the one operated by [NIST](https://beacon.nist.gov/home) or the hash of the latest blocks in a cryptocurrency network could be used to affirm that a block was at least not *created* before a given time. + +# Direct Connections + +We propose a system to \ No newline at end of file diff --git a/onionr/api.py b/onionr/api.py index bf592c59..d0407388 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -18,18 +18,21 @@ along with this program. If not, see . ''' import flask -from flask import request, Response, abort +from flask import request, Response, abort, send_from_directory from multiprocessing import Process from gevent.wsgi import WSGIServer -import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config +import sys, random, threading, hmac, hashlib, base64, time, math, os, json from core import Core from onionrblockapi import Block -import onionrutils, onionrcrypto +import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config class API: ''' Main HTTP API (Flask) ''' + + callbacks = {'public' : {}, 'private' : {}} + def validateToken(self, token): ''' Validate that the client token matches the given token @@ -42,6 +45,30 @@ class API: except TypeError: return False + def guessMime(path): + ''' + Guesses the mime type from the input filename + ''' + + mimetypes = { + 'html' : 'text/html', + 'js' : 'application/javascript', + 'css' : 'text/css', + 'png' : 'image/png', + 'jpg' : 'image/jpeg' + } + + for mimetype in mimetypes: + logger.debug(path + ' endswith .' + mimetype + '?') + if path.endswith('.%s' % mimetype): + logger.debug('- True!') + return mimetypes[mimetype] + else: + logger.debug('- no') + + logger.debug('%s not in %s' % (path, mimetypes)) + return 'text/plain' + def __init__(self, debug): ''' Initialize the api server, preping variables for later use @@ -73,6 +100,7 @@ class API: self.i2pEnabled = config.get('i2p.host', False) self.mimeType = 'text/plain' + self.overrideCSP = False with open('data/time-bypass.txt', 'w') as bypass: bypass.write(self.timeBypassToken) @@ -92,7 +120,6 @@ class API: Simply define the request as not having yet failed, before every request. ''' self.requestFailed = False - return @app.after_request @@ -102,17 +129,85 @@ class API: #else: # resp.headers['server'] = 'Onionr' resp.headers['Content-Type'] = self.mimeType - resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" + if not self.overrideCSP: + resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" resp.headers['X-Frame-Options'] = 'deny' resp.headers['X-Content-Type-Options'] = "nosniff" resp.headers['server'] = 'Onionr' # reset to text/plain to help prevent browser attacks - if self.mimeType != 'text/plain': - self.mimeType = 'text/plain' + self.mimeType = 'text/plain' + self.overrideCSP = False return resp + @app.route('/www/private/') + def www_private(path): + startTime = math.floor(time.time()) + + if request.args.get('timingToken') is None: + timingToken = '' + else: + timingToken = request.args.get('timingToken') + + if not config.get("www.private.run", True): + abort(403) + + self.validateHost('private') + + endTime = math.floor(time.time()) + elapsed = endTime - startTime + + if not hmac.compare_digest(timingToken, self.timeBypassToken): + if elapsed < self._privateDelayTime: + time.sleep(self._privateDelayTime - elapsed) + + return send_from_directory('static-data/www/private/', path) + + @app.route('/www/public/') + def www_public(path): + if not config.get("www.public.run", True): + abort(403) + + self.validateHost('public') + + return send_from_directory('static-data/www/public/', path) + + @app.route('/ui/') + def ui_private(path): + startTime = math.floor(time.time()) + + ''' + if request.args.get('timingToken') is None: + timingToken = '' + else: + timingToken = request.args.get('timingToken') + ''' + + if not config.get("www.ui.run", True): + abort(403) + + if config.get("www.ui.private", True): + self.validateHost('private') + else: + self.validateHost('public') + + ''' + endTime = math.floor(time.time()) + elapsed = endTime - startTime + + if not hmac.compare_digest(timingToken, self.timeBypassToken): + if elapsed < self._privateDelayTime: + time.sleep(self._privateDelayTime - elapsed) + ''' + + logger.debug('Serving %s' % path) + + self.mimeType = API.guessMime(path) + self.overrideCSP = True + + return send_from_directory('static-data/www/ui/dist/', path, mimetype = API.guessMime(path)) + @app.route('/client/') def private_handler(): if request.args.get('timingToken') is None: @@ -132,6 +227,9 @@ class API: if not self.validateToken(token): abort(403) + + events.event('webapi_private', onionr = None, data = {'action' : action, 'data' : data, 'timingToken' : timingToken, 'token' : token}) + self.validateHost('private') if action == 'hello': resp = Response('Hello, World! ' + request.host) @@ -141,17 +239,120 @@ class API: resp = Response('Goodbye') elif action == 'ping': resp = Response('pong') - elif action == 'stats': - resp = Response('me_irl') - raise Exception - elif action == 'site': - block = data - siteData = self._core.getData(data) - response = 'not found' - if siteData != '' and siteData != False: - self.mimeType = 'text/html' - response = siteData.split(b'-', 2)[-1] - resp = Response(response) + elif action == "insertBlock": + response = {'success' : False, 'reason' : 'An unknown error occurred'} + + if not ((data is None) or (len(str(data).strip()) == 0)): + try: + decoded = json.loads(data) + + block = Block() + + sign = False + + for key in decoded: + val = decoded[key] + + key = key.lower() + + if key == 'type': + block.setType(val) + elif key in ['body', 'content']: + block.setContent(val) + elif key == 'parent': + block.setParent(val) + elif key == 'sign': + sign = (str(val).lower() == 'true') + + hash = block.save(sign = sign) + + if not hash is False: + response['success'] = True + response['hash'] = hash + response['reason'] = 'Successfully wrote block to file' + else: + response['reason'] = 'Failed to save the block' + except Exception as e: + logger.warn('insertBlock api request failed', error = e) + logger.debug('Here\'s the request: %s' % data) + else: + response = {'success' : False, 'reason' : 'Missing `data` parameter.', 'blocks' : {}} + + resp = Response(json.dumps(response)) + elif action == 'searchBlocks': + response = {'success' : False, 'reason' : 'An unknown error occurred', 'blocks' : {}} + + if not ((data is None) or (len(str(data).strip()) == 0)): + try: + decoded = json.loads(data) + + type = None + signer = None + signed = None + parent = None + reverse = False + limit = None + + for key in decoded: + val = decoded[key] + + key = key.lower() + + if key == 'type': + type = str(val) + elif key == 'signer': + if isinstance(val, list): + signer = val + else: + signer = str(val) + elif key == 'signed': + signed = (str(val).lower() == 'true') + elif key == 'parent': + parent = str(val) + elif key == 'reverse': + reverse = (str(val).lower() == 'true') + elif key == 'limit': + limit = 10000 + + if val is None: + val = limit + + limit = min(limit, int(val)) + + blockObjects = Block.getBlocks(type = type, signer = signer, signed = signed, parent = parent, reverse = reverse, limit = limit) + + logger.debug('%s results for query %s' % (len(blockObjects), decoded)) + + blocks = list() + + for block in blockObjects: + blocks.append({ + 'hash' : block.getHash(), + 'type' : block.getType(), + 'content' : block.getContent(), + 'signature' : block.getSignature(), + 'signedData' : block.getSignedData(), + 'signed' : block.isSigned(), + 'valid' : block.isValid(), + 'date' : (int(block.getDate().strftime("%s")) if not block.getDate() is None else None), + 'parent' : (block.getParent().getHash() if not block.getParent() is None else None), + 'metadata' : block.getMetadata(), + 'header' : block.getHeader() + }) + + response['success'] = True + response['blocks'] = blocks + response['reason'] = 'Success' + except Exception as e: + logger.warn('searchBlock api request failed', error = e) + logger.debug('Here\'s the request: %s' % data) + else: + response = {'success' : False, 'reason' : 'Missing `data` parameter.', 'blocks' : {}} + + resp = Response(json.dumps(response)) + + elif action in API.callbacks['private']: + resp = Response(str(getCallback(action, scope = 'private')(request))) else: resp = Response('(O_o) Dude what? (invalid command)') endTime = math.floor(time.time()) @@ -175,6 +376,68 @@ class API: resp = Response("") return resp + @app.route('/public/upload/', methods=['POST']) + def blockUpload(): + self.validateHost('public') + resp = 'failure' + try: + data = request.form['block'] + except KeyError: + logger.warn('No block specified for upload') + pass + else: + if sys.getsizeof(data) < 100000000: + try: + if blockimporter.importBlockFromData(data, self._core): + resp = 'success' + else: + logger.warn('Error encountered importing uploaded block') + except onionrexceptions.BlacklistedBlock: + logger.debug('uploaded block is blacklisted') + pass + + resp = Response(resp) + return resp + + @app.route('/public/announce/', methods=['POST']) + def acceptAnnounce(): + self.validateHost('public') + resp = 'failure' + powHash = '' + randomData = '' + newNode = '' + ourAdder = self._core.hsAddress.encode() + try: + newNode = request.form['node'].encode() + except KeyError: + logger.warn('No block specified for upload') + pass + else: + try: + randomData = request.form['random'] + randomData = base64.b64decode(randomData) + except KeyError: + logger.warn('No random data specified for upload') + else: + nodes = newNode + self._core.hsAddress.encode() + nodes = self._core._crypto.blake2bHash(nodes) + powHash = self._core._crypto.blake2bHash(randomData + nodes) + try: + powHash = powHash.decode() + except AttributeError: + pass + if powHash.startswith('0000'): + try: + newNode = newNode.decode() + except AttributeError: + pass + if self._core.addAddress(newNode): + resp = 'Success' + else: + logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash) + resp = Response(resp) + return resp + @app.route('/public/') def public_handler(): # Public means it is publicly network accessible @@ -186,6 +449,9 @@ class API: data = data except: data = '' + + events.event('webapi_public', onionr = None, data = {'action' : action, 'data' : data, 'requestingPeer' : requestingPeer, 'request' : request}) + if action == 'firstConnect': pass elif action == 'ping': @@ -196,22 +462,11 @@ class API: resp = Response(self._utils.getBlockDBHash()) elif action == 'getBlockHashes': resp = Response('\n'.join(self._core.getBlockList())) - elif action == 'directMessage': - resp = Response(self._core.handle_direct_connection(data)) - elif action == 'announce': - if data != '': - # TODO: require POW for this - if self._core.addAddress(data): - resp = Response('Success') - else: - resp = Response('') - else: - resp = Response('') # setData should be something the communicator initiates, not this api elif action == 'getData': resp = '' if self._utils.validateHash(data): - if not os.path.exists('data/blocks/' + data + '.db'): + if os.path.exists('data/blocks/' + data + '.dat'): block = Block(hash=data.encode(), core=self._core) resp = base64.b64encode(block.getRaw().encode()).decode() if len(resp) == 0: @@ -227,6 +482,8 @@ class API: peers = self._core.listPeers(getPow=True) response = ','.join(peers) resp = Response(response) + elif action in API.callbacks['public']: + resp = Response(str(getCallback(action, scope = 'public')(request))) else: resp = Response("") @@ -243,7 +500,6 @@ class API: def authFail(err): self.requestFailed = True resp = Response("403") - return resp @app.errorhandler(401) @@ -256,11 +512,13 @@ class API: logger.info('Starting client on ' + self.host + ':' + str(bindPort) + '...', timestamp=False) try: + while len(self._core.hsAddress) == 0: + self._core.refreshFirstStartVars() + time.sleep(0.5) self.http_server = WSGIServer((self.host, bindPort), app) self.http_server.serve_forever() except KeyboardInterrupt: pass - #app.run(host=self.host, port=bindPort, debug=False, threaded=True) except Exception as e: logger.error(str(e)) logger.fatal('Failed to start client on ' + self.host + ':' + str(bindPort) + ', exiting...') @@ -297,3 +555,31 @@ class API: # we exit rather than abort to avoid fingerprinting logger.debug('Avoiding fingerprinting, exiting...') sys.exit(1) + + def setCallback(action, callback, scope = 'public'): + if not scope in API.callbacks: + return False + + API.callbacks[scope][action] = callback + + return True + + def removeCallback(action, scope = 'public'): + if (not scope in API.callbacks) or (not action in API.callbacks[scope]): + return False + + del API.callbacks[scope][action] + + return True + + def getCallback(action, scope = 'public'): + if (not scope in API.callbacks) or (not action in API.callbacks[scope]): + return None + + return API.callbacks[scope][action] + + def getCallbacks(scope = None): + if (not scope is None) and (scope in API.callbacks): + return API.callbacks[scope] + + return API.callbacks diff --git a/onionr/blockimporter.py b/onionr/blockimporter.py new file mode 100644 index 00000000..2c29927f --- /dev/null +++ b/onionr/blockimporter.py @@ -0,0 +1,46 @@ +''' + Onionr - P2P Microblogging Platform & Social network + + Import block data and save it +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import core, onionrexceptions, logger +def importBlockFromData(content, coreInst): + retData = False + + dataHash = coreInst._crypto.sha3Hash(content) + + if coreInst._blacklist.inBlacklist(dataHash): + raise onionrexceptions.BlacklistedBlock('%s is a blacklisted block' % (dataHash,)) + + if not isinstance(coreInst, core.Core): + raise Exception("coreInst must be an Onionr core instance") + + try: + content = content.encode() + except AttributeError: + pass + + metas = coreInst._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata + metadata = metas[0] + if coreInst._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid + if coreInst._crypto.verifyPow(content): # check if POW is enough/correct + logger.info('Block passed proof, saving.') + blockHash = coreInst.setData(content) + coreInst.addToBlockDB(blockHash, dataSaved=True) + coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database + retData = True + return retData \ No newline at end of file diff --git a/onionr/communicator2.py b/onionr/communicator2.py index b4a32649..42be9e45 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -19,8 +19,9 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import sys, os, core, config, json, onionrblockapi as block, requests, time, logger, threading, onionrplugins as plugins, base64, onionr -import onionrexceptions +import sys, os, core, config, json, requests, time, logger, threading, base64, onionr +import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block +import onionrdaemontools from defusedxml import minidom class OnionrCommunicatorDaemon: @@ -37,6 +38,8 @@ class OnionrCommunicatorDaemon: self.nistSaltTimestamp = 0 self.powSalt = 0 + self.blockToUpload = '' + # loop time.sleep delay in seconds self.delay = 1 @@ -46,6 +49,7 @@ class OnionrCommunicatorDaemon: # lists of connected peers and peers we know we can't reach currently self.onlinePeers = [] self.offlinePeers = [] + self.peerProfiles = [] # list of peer's profiles (onionrpeers.PeerProfile instances) # amount of threads running by name, used to prevent too many self.threadCounts = {} @@ -66,6 +70,11 @@ class OnionrCommunicatorDaemon: # Loads in and starts the enabled plugins plugins.reload() + # daemon tools are misc daemon functions, e.g. announce to online peers + # intended only for use by OnionrCommunicatorDaemon + #self.daemonTools = onionrdaemontools.DaemonTools(self) + self.daemonTools = onionrdaemontools.DaemonTools(self) + if debug or developmentMode: OnionrCommunicatorTimers(self, self.heartbeat, 10) @@ -74,17 +83,23 @@ class OnionrCommunicatorDaemon: self.header() # Set timers, function reference, seconds + # requiresPeer True means the timer function won't fire if we have no connected peers + # TODO: make some of these timer counts configurable OnionrCommunicatorTimers(self, self.daemonCommands, 5) OnionrCommunicatorTimers(self, self.detectAPICrash, 5) peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60) - OnionrCommunicatorTimers(self, self.lookupBlocks, 7, requiresPeer=True) + OnionrCommunicatorTimers(self, self.lookupBlocks, 7, requiresPeer=True, maxThreads=1) OnionrCommunicatorTimers(self, self.getBlocks, 10, requiresPeer=True) OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58) OnionrCommunicatorTimers(self, self.lookupKeys, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) + announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 305, requiresPeer=True, maxThreads=1) + cleanupTimer = OnionrCommunicatorTimers(self, self.peerCleanup, 300, requiresPeer=True) # set loop to execute instantly to load up peer pool (replaced old pool init wait) - peerPoolTimer.count = (peerPoolTimer.frequency - 1) + peerPoolTimer.count = (peerPoolTimer.frequency - 1) + cleanupTimer.count = (cleanupTimer.frequency - 60) + announceTimer.count = (cleanupTimer.frequency - 60) # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking try: @@ -101,7 +116,7 @@ class OnionrCommunicatorDaemon: logger.info('Goodbye.') self._core._utils.localCommand('shutdown') time.sleep(0.5) - + def lookupKeys(self): '''Lookup new keys''' logger.debug('Looking up new keys...') @@ -111,7 +126,6 @@ class OnionrCommunicatorDaemon: peer = self.pickOnlinePeer() newKeys = self.peerAction(peer, action='kex') self._core._utils.mergeKeys(newKeys) - self.decrementThreadCount('lookupKeys') return @@ -132,14 +146,26 @@ class OnionrCommunicatorDaemon: tryAmount = 2 newBlocks = '' existingBlocks = self._core.getBlockList() + triedPeers = [] # list of peers we've tried this time around for i in range(tryAmount): peer = self.pickOnlinePeer() # select random online peer + # if we've already tried all the online peers this time around, stop + if peer in triedPeers: + if len(self.onlinePeers) == len(triedPeers): + break + else: + continue newDBHash = self.peerAction(peer, 'getDBHash') # get their db hash if newDBHash == False: continue # if request failed, restart loop (peer is added to offline peers automatically) + triedPeers.append(peer) if newDBHash != self._core.getAddressInfo(peer, 'DBHash'): self._core.setAddressInfo(peer, 'DBHash', newDBHash) - newBlocks = self.peerAction(peer, 'getBlockHashes') + try: + newBlocks = self.peerAction(peer, 'getBlockHashes') + except Exception as error: + logger.warn("could not get new blocks with " + peer, error=error) + newBlocks = False if newBlocks != False: # if request was a success for i in newBlocks.split('\n'): @@ -147,7 +173,7 @@ class OnionrCommunicatorDaemon: # if newline seperated string is valid hash if not i in existingBlocks: # if block does not exist on disk and is not already in block queue - if i not in self.blockQueue: + if i not in self.blockQueue and not self._core._blacklist.inBlacklist(i): self.blockQueue.append(i) self.decrementThreadCount('lookupBlocks') return @@ -155,12 +181,19 @@ class OnionrCommunicatorDaemon: def getBlocks(self): '''download new blocks in queue''' for blockHash in self.blockQueue: + if self.shutdown: + break if blockHash in self.currentDownloading: logger.debug('ALREADY DOWNLOADING ' + blockHash) continue + if blockHash in self._core.getBlockList(): + logger.debug('%s is already saved' % (blockHash,)) + self.blockQueue.remove(blockHash) + continue self.currentDownloading.append(blockHash) logger.info("Attempting to download %s..." % blockHash) - content = self.peerAction(self.pickOnlinePeer(), 'getData', data=blockHash) # block content from random peer (includes metadata) + peerUsed = self.pickOnlinePeer() + content = self.peerAction(peerUsed, 'getData', data=blockHash) # block content from random peer (includes metadata) if content != False: try: content = content.encode() @@ -177,7 +210,7 @@ class OnionrCommunicatorDaemon: metas = self._core._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata metadata = metas[0] #meta = metas[1] - if self._core._utils.validateMetadata(metadata): # check if metadata is valid + if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce if self._core._crypto.verifyPow(content): # check if POW is enough/correct logger.info('Block passed proof, saving.') self._core.setData(content) @@ -186,7 +219,11 @@ class OnionrCommunicatorDaemon: else: logger.warn('POW failed for block ' + blockHash) else: - logger.warn('Metadata for ' + blockHash + ' is invalid.') + if self._core._blacklist.inBlacklist(realHash): + logger.warn('%s is blacklisted' % (realHash,)) + else: + logger.warn('Metadata for ' + blockHash + ' is invalid.') + self._core._blacklist.addToDB(blockHash) else: # if block didn't meet expected hash tempHash = self._core._crypto.sha3Hash(content) # lazy hack, TODO use var @@ -194,9 +231,11 @@ class OnionrCommunicatorDaemon: tempHash = tempHash.decode() except AttributeError: pass + # Punish peer for sharing invalid block (not always malicious, but is bad regardless) + onionrpeers.PeerProfiles(peerUsed, self._core).addScore(-50) logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash) self.blockQueue.remove(blockHash) # remove from block queue both if success or false - self.currentDownloading.remove(blockHash) + self.currentDownloading.remove(blockHash) self.decrementThreadCount('getBlocks') return @@ -238,12 +277,14 @@ class OnionrCommunicatorDaemon: '''Manages the self.onlinePeers attribute list, connects to more peers if we have none connected''' logger.info('Refreshing peer pool.') - maxPeers = 4 + maxPeers = 6 needed = maxPeers - len(self.onlinePeers) for i in range(needed): if len(self.onlinePeers) == 0: self.connectNewPeer(useBootstrap=True) + else: + self.connectNewPeer() if self.shutdown: break else: @@ -254,8 +295,9 @@ class OnionrCommunicatorDaemon: def addBootstrapListToPeerList(self, peerList): '''Add the bootstrap list to the peer list (no duplicates)''' for i in self._core.bootstrapList: - if i not in peerList and i not in self.offlinePeers and i != self._core.hsAdder: + if i not in peerList and i not in self.offlinePeers and i != self._core.hsAddress: peerList.append(i) + self._core.addAddress(i) def connectNewPeer(self, peer='', useBootstrap=False): '''Adds a new random online peer to self.onlinePeers''' @@ -268,6 +310,8 @@ class OnionrCommunicatorDaemon: raise onionrexceptions.InvalidAddress('Will not attempt connection test to invalid address') else: peerList = self._core.listAdders() + + peerList = onionrpeers.getScoreSortedPeerList(self._core) if len(peerList) == 0 or useBootstrap: # Avoid duplicating bootstrap addresses in peerList @@ -276,18 +320,32 @@ class OnionrCommunicatorDaemon: for address in peerList: if len(address) == 0 or address in tried or address in self.onlinePeers: continue + if self.shutdown: + return if self.peerAction(address, 'ping') == 'pong!': logger.info('Connected to ' + address) time.sleep(0.1) if address not in self.onlinePeers: self.onlinePeers.append(address) retData = address + + # add peer to profile list if they're not in it + for profile in self.peerProfiles: + if profile.address == address: + break + else: + self.peerProfiles.append(onionrpeers.PeerProfiles(address, self._core)) break else: tried.append(address) logger.debug('Failed to connect to ' + address) return retData + def peerCleanup(self): + '''This just calls onionrpeers.cleanupPeers, which removes dead or bad peers (offline too long, too slow)''' + onionrpeers.peerCleanup(self._core) + self.decrementThreadCount('peerCleanup') + def printOnlinePeers(self): '''logs online peer list''' if len(self.onlinePeers) == 0: @@ -295,7 +353,8 @@ class OnionrCommunicatorDaemon: else: logger.info('Online peers:') for i in self.onlinePeers: - logger.info(i) + score = str(self.getPeerProfileInstance(i).score) + logger.info(i + ', score: ' + score) def peerAction(self, peer, action, data=''): '''Perform a get request to a peer''' @@ -305,14 +364,33 @@ class OnionrCommunicatorDaemon: url = 'http://' + peer + '/public/?action=' + action if len(data) > 0: url += '&data=' + data + + self._core.setAddressInfo(peer, 'lastConnectAttempt', self._core._utils.getEpoch()) # mark the time we're trying to request this peer + retData = self._core._utils.doGetRequest(url, port=self.proxyPort) # if request failed, (error), mark peer offline if retData == False: try: + self.getPeerProfileInstance(peer).addScore(-10) self.onlinePeers.remove(peer) self.getOnlinePeers() # Will only add a new peer to pool if needed except ValueError: pass + else: + self._core.setAddressInfo(peer, 'lastConnect', self._core._utils.getEpoch()) + self.getPeerProfileInstance(peer).addScore(1) + return retData + + def getPeerProfileInstance(self, peer): + '''Gets a peer profile instance from the list of profiles, by address name''' + for i in self.peerProfiles: + # if the peer's profile is already loaded, return that + if i.address == peer: + retData = i + break + else: + # if the peer's profile is not loaded, return a new one. connectNewPeer adds it the list on connect + retData = onionrpeers.PeerProfiles(peer, self._core) return retData def heartbeat(self): @@ -326,6 +404,8 @@ class OnionrCommunicatorDaemon: cmd = self._core.daemonQueue() if cmd is not False: + events.event('daemon_command', onionr = None, data = {'cmd' : cmd}) + if cmd[0] == 'shutdown': self.shutdown = True elif cmd[0] == 'announceNode': @@ -339,23 +419,46 @@ class OnionrCommunicatorDaemon: for i in self.timers: if i.timerFunction.__name__ == 'lookupKeys': i.count = (i.frequency - 1) + elif cmd[0] == 'pex': + for i in self.timers: + if i.timerFunction.__name__ == 'lookupAdders': + i.count = (i.frequency - 1) + elif cmd[0] == 'uploadBlock': + self.blockToUpload = cmd[1] + threading.Thread(target=self.uploadBlock).start() else: logger.info('Recieved daemonQueue command:' + cmd[0]) + self.decrementThreadCount('daemonCommands') + def uploadBlock(self): + '''Upload our block to a few peers''' + # when inserting a block, we try to upload it to a few peers to add some deniability + triedPeers = [] + if not self._core._utils.validateHash(self.blockToUpload): + logger.warn('Requested to upload invalid block') + return + for i in range(max(len(self.onlinePeers), 2)): + peer = self.pickOnlinePeer() + if peer in triedPeers: + continue + triedPeers.append(peer) + url = 'http://' + peer + '/public/upload/' + data = {'block': block.Block(self.blockToUpload).getRaw()} + proxyType = '' + if peer.endswith('.onion'): + proxyType = 'tor' + elif peer.endswith('.i2p'): + proxyType = 'i2p' + logger.info("Uploading block") + self._core._utils.doPostRequest(url, data=data, proxyType=proxyType) + def announce(self, peer): '''Announce to peers our address''' - announceCount = 0 - announceAmount = 2 - for peer in self.onlinePeers: - announceCount += 1 - if self.peerAction(peer, 'announce', self._core.hsAdder) == 'Success': - logger.info('Successfully introduced node to ' + peer) - break - else: - if announceCount == announceAmount: - logger.warn('Could not introduce node. Try again soon') - break + if self.daemonTools.announceNode(): + logger.info('Successfully introduced node to ' + peer) + else: + logger.warn('Could not introduce node.') def detectAPICrash(self): '''exit if the api server crashes/stops''' @@ -366,6 +469,7 @@ class OnionrCommunicatorDaemon: time.sleep(1) else: # This executes if the api is NOT detected to be running + events.event('daemon_crash', onionr = None, data = {}) logger.error('Daemon detected API crash (or otherwise unable to reach API after long time), stopping...') self.shutdown = True self.decrementThreadCount('detectAPICrash') diff --git a/onionr/core.py b/onionr/core.py index fbe9cc80..9d97e3f1 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -21,7 +21,8 @@ import sqlite3, os, sys, time, math, base64, tarfile, getpass, simplecrypt, hash from onionrblockapi import Block import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues - +import onionrblacklist +import dbcreator if sys.version_info < (3, 6): try: import sha3 @@ -40,11 +41,15 @@ class Core: self.blockDB = 'data/blocks.db' self.blockDataLocation = 'data/blocks/' self.addressDB = 'data/address.db' - self.hsAdder = '' - + self.hsAddress = '' self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt' self.bootstrapList = [] self.requirements = onionrvalues.OnionrValues() + self.torPort = torPort + self.dataNonceFile = 'data/block-nonces.dat' + self.dbCreate = dbcreator.DBCreator(self) + + self.usageFile = 'data/disk-usage.txt' if not os.path.exists('data/'): os.mkdir('data/') @@ -55,7 +60,7 @@ class Core: if os.path.exists('data/hs/hostname'): with open('data/hs/hostname', 'r') as hs: - self.hsAdder = hs.read().strip() + self.hsAddress = hs.read().strip() # Load bootstrap address list if os.path.exists(self.bootstrapFileLocation): @@ -69,6 +74,7 @@ class Core: self._utils = onionrutils.OnionrUtils(self) # Initialize the crypto object self._crypto = onionrcrypto.OnionrCrypto(self) + self._blacklist = onionrblacklist.OnionrBlackList(self) except Exception as error: logger.error('Failed to initialize core Onionr library.', error=error) @@ -76,6 +82,12 @@ class Core: sys.exit(1) return + def refreshFirstStartVars(self): + '''Hack to refresh some vars which may not be set on first start''' + if os.path.exists('data/hs/hostname'): + with open('data/hs/hostname', 'r') as hs: + self.hsAddress = hs.read().strip() + def addPeer(self, peerID, powID, name=''): ''' Adds a public key to the key database (misleading function name) @@ -123,7 +135,6 @@ class Core: for i in c.execute("SELECT * FROM adders where address = '" + address + "';"): try: if i[0] == address: - logger.warn('Not adding existing address') conn.close() return False except ValueError: @@ -156,14 +167,13 @@ class Core: conn.close() events.event('address_remove', data = {'address': address}, onionr = None) - return True else: return False def removeBlock(self, block): ''' - remove a block from this node + remove a block from this node (does not automatically blacklist) ''' if self._utils.validateHash(block): conn = sqlite3.connect(self.blockDB) @@ -180,87 +190,20 @@ class Core: def createAddressDB(self): ''' Generate the address database - - types: - 1: I2P b32 address - 2: Tor v2 (like facebookcorewwwi.onion) - 3: Tor v3 ''' - conn = sqlite3.connect(self.addressDB) - c = conn.cursor() - c.execute('''CREATE TABLE adders( - address text, - type int, - knownPeer text, - speed int, - success int, - DBHash text, - powValue text, - failure int, - lastConnect int - ); - ''') - conn.commit() - conn.close() + self.dbCreate.createAddressDB() def createPeerDB(self): ''' Generate the peer sqlite3 database and populate it with the peers table. ''' - # generate the peer database - conn = sqlite3.connect(self.peerDB) - c = conn.cursor() - c.execute('''CREATE TABLE peers( - ID text not null, - name text, - adders text, - blockDBHash text, - forwardKey text, - dateSeen not null, - bytesStored int, - trust int, - pubkeyExchanged int, - hashID text, - pow text not null); - ''') - conn.commit() - conn.close() - return + self.dbCreate.createPeerDB() def createBlockDB(self): ''' Create a database for blocks - - hash - the hash of a block - dateReceived - the date the block was recieved, not necessarily when it was created - decrypted - if we can successfully decrypt the block (does not describe its current state) - dataType - data type of the block - dataFound - if the data has been found for the block - dataSaved - if the data has been saved for the block - sig - optional signature by the author (not optional if author is specified) - author - multi-round partial sha3-256 hash of authors public key - dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is ''' - if os.path.exists(self.blockDB): - raise Exception("Block database already exists") - conn = sqlite3.connect(self.blockDB) - c = conn.cursor() - c.execute('''CREATE TABLE hashes( - hash text not null, - dateReceived int, - decrypted int, - dataType text, - dataFound int, - dataSaved int, - sig text, - author text, - dateClaimed int - ); - ''') - conn.commit() - conn.close() - - return + self.dbCreate.createBlockDB() def addToBlockDB(self, newHash, selfInsert=False, dataSaved=False): ''' @@ -300,16 +243,24 @@ class Core: return data - def setData(self, data): - ''' - Set the data assciated with a hash - ''' - data = data + def _getSha3Hash(self, data): hasher = hashlib.sha3_256() if not type(data) is bytes: data = data.encode() hasher.update(data) dataHash = hasher.hexdigest() + return dataHash + + def setData(self, data): + ''' + Set the data assciated with a hash + ''' + data = data + if not type(data) is bytes: + data = data.encode() + + dataHash = self._getSha3Hash(data) + if type(dataHash) is bytes: dataHash = dataHash.decode() blockFileName = self.blockDataLocation + dataHash + '.dat' @@ -571,30 +522,12 @@ class Core: c = conn.cursor() command = (data, address) # TODO: validate key on whitelist - if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'DBHash', 'failure', 'lastConnect'): + if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'DBHash', 'failure', 'lastConnect', 'lastConnectAttempt'): raise Exception("Got invalid database key when setting address info") - c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command) - conn.commit() - conn.close() - return - - def handle_direct_connection(self, data): - ''' - Handles direct messages - ''' - try: - data = json.loads(data) - - # TODO: Determine the sender, verify, etc - if ('callback' in data) and (data['callback'] is True): - # then this is a response to the message we sent earlier - self.daemonQueueAdd('checkCallbacks', json.dumps(data)) - else: - # then we should handle it and respond accordingly - self.daemonQueueAdd('incomingDirectConnection', json.dumps(data)) - except Exception as e: - logger.warn('Failed to handle incoming direct message: %s' % str(e)) - + else: + c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command) + conn.commit() + conn.close() return def getBlockList(self, unsaved = False): # TODO: Use unsaved?? @@ -606,7 +539,7 @@ class Core: if unsaved: execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();' else: - execute = 'SELECT hash FROM hashes ORDER BY RANDOM();' + execute = 'SELECT hash FROM hashes ORDER BY dateReceived DESC;' rows = list() for row in c.execute(execute): for i in row: @@ -661,7 +594,7 @@ class Core: def updateBlockInfo(self, hash, key, data): ''' sets info associated with a block - + hash - the hash of a block dateReceived - the date the block was recieved, not necessarily when it was created decrypted - if we can successfully decrypt the block (does not describe its current state) @@ -691,6 +624,18 @@ class Core: ''' retData = False + # check nonce + dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data)) + try: + with open(self.dataNonceFile, 'r') as nonces: + if dataNonce in nonces: + return retData + except FileNotFoundError: + pass + # record nonce + with open(self.dataNonceFile, 'a') as nonceFile: + nonceFile.write(dataNonce + '\n') + if meta is None: meta = dict() @@ -702,6 +647,7 @@ class Core: signature = '' signer = '' metadata = {} + # metadata is full block metadata, meta is internal, user specified metadata # only use header if not set in provided meta if not header is None: @@ -743,13 +689,13 @@ class Core: signer = self._crypto.pubKeyEncrypt(signer, asymPeer, encodedData=True, anonymous=True).decode() else: raise onionrexceptions.InvalidPubkey(asymPeer + ' is not a valid base32 encoded ed25519 key') - + # compile metadata metadata['meta'] = jsonMeta metadata['sig'] = signature metadata['signer'] = signer metadata['time'] = str(self._utils.getEpoch()) - + # send block data (and metadata) to POW module to get tokenized block data proof = onionrproofs.POW(metadata, data) payload = proof.waitForResult() @@ -757,6 +703,7 @@ class Core: retData = self.setData(payload) self.addToBlockDB(retData, selfInsert=True, dataSaved=True) self.setBlockType(retData, meta['type']) + self.daemonQueueAdd('uploadBlock', retData) if retData != False: events.event('insertBlock', onionr = None, threaded = False) diff --git a/onionr/dbcreator.py b/onionr/dbcreator.py new file mode 100644 index 00000000..19a9a7bd --- /dev/null +++ b/onionr/dbcreator.py @@ -0,0 +1,109 @@ +''' + Onionr - P2P Anonymous Data Storage & Sharing + + DBCreator, creates sqlite3 databases used by Onionr +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import sqlite3, os +class DBCreator: + def __init__(self, coreInst): + self.core = coreInst + + def createAddressDB(self): + ''' + Generate the address database + + types: + 1: I2P b32 address + 2: Tor v2 (like facebookcorewwwi.onion) + 3: Tor v3 + ''' + conn = sqlite3.connect(self.core.addressDB) + c = conn.cursor() + c.execute('''CREATE TABLE adders( + address text, + type int, + knownPeer text, + speed int, + success int, + DBHash text, + powValue text, + failure int, + lastConnect int, + lastConnectAttempt int, + trust int + ); + ''') + conn.commit() + conn.close() + + def createPeerDB(self): + ''' + Generate the peer sqlite3 database and populate it with the peers table. + ''' + # generate the peer database + conn = sqlite3.connect(self.core.peerDB) + c = conn.cursor() + c.execute('''CREATE TABLE peers( + ID text not null, + name text, + adders text, + blockDBHash text, + forwardKey text, + dateSeen not null, + bytesStored int, + trust int, + pubkeyExchanged int, + hashID text, + pow text not null); + ''') + conn.commit() + conn.close() + return + + def createBlockDB(self): + ''' + Create a database for blocks + + hash - the hash of a block + dateReceived - the date the block was recieved, not necessarily when it was created + decrypted - if we can successfully decrypt the block (does not describe its current state) + dataType - data type of the block + dataFound - if the data has been found for the block + dataSaved - if the data has been saved for the block + sig - optional signature by the author (not optional if author is specified) + author - multi-round partial sha3-256 hash of authors public key + dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is + ''' + if os.path.exists(self.core.blockDB): + raise Exception("Block database already exists") + conn = sqlite3.connect(self.core.blockDB) + c = conn.cursor() + c.execute('''CREATE TABLE hashes( + hash text not null, + dateReceived int, + decrypted int, + dataType text, + dataFound int, + dataSaved int, + sig text, + author text, + dateClaimed int + ); + ''') + conn.commit() + conn.close() + return \ No newline at end of file diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index 7ad74cc0..19f355e4 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -97,10 +97,10 @@ DataDirectory data/tordata/ elif 'Opening Socks listener' in line.decode(): logger.debug(line.decode().replace('\n', '')) else: - logger.fatal('Failed to start Tor. Try killing any other Tor processes owned by this user.') + logger.fatal('Failed to start Tor. Maybe a stray instance of Tor used by Onionr is still running?') return False except KeyboardInterrupt: - logger.fatal("Got keyboard interrupt") + logger.fatal("Got keyboard interrupt.") return False logger.debug('Finished starting Tor.', timestamp=True) diff --git a/onionr/onionr.py b/onionr/onionr.py index 10aa54ce..1736c3f9 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -40,9 +40,9 @@ except ImportError: raise Exception("You need the PySocks module (for use with socks5 proxy to use Tor)") ONIONR_TAGLINE = 'Anonymous P2P Platform - GPLv3 - https://Onionr.VoidNet.Tech' -ONIONR_VERSION = '0.1.0' # for debugging and stuff +ONIONR_VERSION = '0.2.0' # for debugging and stuff ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION) -API_VERSION = '4' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes knows how to communicate without learning too much information about you. +API_VERSION = '4' # increments of 1; only change when something fundemental about how the API works changes. This way other nodes know how to communicate without learning too much information about you. class Onionr: def __init__(self): @@ -91,8 +91,6 @@ class Onionr: self.onionrCore = core.Core() self.onionrUtils = OnionrUtils(self.onionrCore) - self.userOS = platform.system() - # Handle commands self.debug = False # Whole application debugging @@ -137,18 +135,18 @@ class Onionr: self.onionrCore.createAddressDB() # Get configuration - - if not data_exists: - # Generate default config - # Hostname should only be set if different from 127.x.x.x. Important for DNS rebinding attack prevention. - if self.debug: - randomPort = 8080 - else: - while True: - randomPort = random.randint(1024, 65535) - if self.onionrUtils.checkPort(randomPort): - break - config.set('client', {'participate': True, 'hmac': base64.b16encode(os.urandom(32)).decode('utf-8'), 'port': randomPort, 'api_version': API_VERSION}, True) + if type(config.get('client.hmac')) is type(None): + config.set('client.hmac', base64.b16encode(os.urandom(32)).decode('utf-8'), savefile=True) + if type(config.get('client.port')) is type(None): + randomPort = 0 + while randomPort < 1024: + randomPort = self.onionrCore._crypto.secrets.randbelow(65535) + config.set('client.port', randomPort, savefile=True) + if type(config.get('client.participate')) is type(None): + config.set('client.participate', True, savefile=True) + if type(config.get('client.api_version')) is type(None): + config.set('client.api_version', API_VERSION, savefile=True) + self.cmds = { '': self.showHelpSuggestion, @@ -188,6 +186,8 @@ class Onionr: 'addaddress': self.addAddress, 'list-peers': self.listPeers, + 'blacklist-block': self.banBlock, + 'add-file': self.addFile, 'addfile': self.addFile, 'listconn': self.listConn, @@ -198,8 +198,19 @@ class Onionr: 'introduce': self.onionrCore.introduceNode, 'connect': self.addAddress, 'kex': self.doKEX, + 'pex': self.doPEX, - 'getpassword': self.getWebPassword + 'ui' : self.openUI, + 'gui' : self.openUI, + + 'getpassword': self.printWebPassword, + 'get-password': self.printWebPassword, + 'getpwd': self.printWebPassword, + 'get-pwd': self.printWebPassword, + 'getpass': self.printWebPassword, + 'get-pass': self.printWebPassword, + 'getpasswd': self.printWebPassword, + 'get-passwd': self.printWebPassword } self.cmdhelp = { @@ -209,7 +220,7 @@ class Onionr: 'start': 'Starts the Onionr daemon', 'stop': 'Stops the Onionr daemon', 'stats': 'Displays node statistics', - 'getpassword': 'Displays the web password', + 'get-password': 'Displays the web password', 'enable-plugin': 'Enables and starts a plugin', 'disable-plugin': 'Disables and stops a plugin', 'reload-plugin': 'Reloads a plugin', @@ -220,6 +231,8 @@ class Onionr: 'import-blocks': 'import blocks from the disk (Onionr is transport-agnostic!)', 'listconn': 'list connected peers', 'kex': 'exchange keys with peers (done automatically)', + 'pex': 'exchange addresses with peers (done automatically)', + 'blacklist-block': 'deletes a block by hash and permanently removes it from your node', 'introduce': 'Introduce your node to the public Onionr network', } @@ -248,6 +261,26 @@ class Onionr: def getCommands(self): return self.cmds + def banBlock(self): + try: + ban = sys.argv[2] + except IndexError: + ban = logger.readline('Enter a block hash:') + if self.onionrUtils.validateHash(ban): + if not self.onionrCore._blacklist.inBlacklist(ban): + try: + self.onionrCore._blacklist.addToDB(ban) + self.onionrCore.removeBlock(ban) + except Exception as error: + logger.error('Could not blacklist block', error=error) + else: + logger.info('Block blacklisted') + else: + logger.warn('That block is already blacklisted') + else: + logger.error('Invalid block hash') + return + def listConn(self): self.onionrCore.daemonQueueAdd('connectedPeers') @@ -259,6 +292,9 @@ class Onionr: def getWebPassword(self): return config.get('client.hmac') + def printWebPassword(self): + print(self.getWebPassword()) + def getHelp(self): return self.cmdhelp @@ -328,6 +364,11 @@ class Onionr: logger.info('Sending kex to command queue...') self.onionrCore.daemonQueueAdd('kex') + def doPEX(self): + '''make communicator do pex''' + logger.info('Sending pex to command queue...') + self.onionrCore.daemonQueueAdd('pex') + def listKeys(self): ''' Displays a list of keys (used to be called peers) (?) @@ -525,22 +566,36 @@ class Onionr: Starts the Onionr communication daemon ''' communicatorDaemon = './communicator2.py' - if not os.environ.get("WERKZEUG_RUN_MAIN") == "true": - if self._developmentMode: - logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)', timestamp = False) - net = NetController(config.get('client.port', 59496)) - logger.info('Tor is starting...') - if not net.startTor(): - sys.exit(1) - logger.info('Started .onion service: ' + logger.colors.underline + net.myID) - logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey) - time.sleep(1) - #TODO make runable on windows - subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)]) - logger.debug('Started communicator') - events.event('daemon_start', onionr = self) - api.API(self.debug) + apiThread = Thread(target=api.API, args=(self.debug,)) + apiThread.start() + try: + time.sleep(3) + except KeyboardInterrupt: + logger.info('Got keyboard interrupt') + time.sleep(1) + self.onionrUtils.localCommand('shutdown') + else: + if apiThread.isAlive(): + if self._developmentMode: + logger.warn('DEVELOPMENT MODE ENABLED (THIS IS LESS SECURE!)', timestamp = False) + net = NetController(config.get('client.port', 59496)) + logger.info('Tor is starting...') + if not net.startTor(): + sys.exit(1) + logger.info('Started .onion service: ' + logger.colors.underline + net.myID) + logger.info('Our Public key: ' + self.onionrCore._crypto.pubKey) + time.sleep(1) + #TODO make runable on windows + subprocess.Popen([communicatorDaemon, "run", str(net.socksPort)]) + logger.debug('Started communicator') + events.event('daemon_start', onionr = self) + try: + while True: + time.sleep(5) + except KeyboardInterrupt: + self.onionrCore.daemonQueueAdd('shutdown') + self.onionrUtils.localCommand('shutdown') return def killDaemon(self): @@ -697,5 +752,12 @@ class Onionr: else: logger.error('%s add-file ' % sys.argv[0], timestamp = False) + def openUI(self): + import webbrowser + url = 'http://127.0.0.1:%s/ui/index.html?timingToken=%s' % (config.get('client.port', 59496), self.onionrUtils.getTimeBypassToken()) + + print('Opening %s ...' % url) + webbrowser.open(url, new = 1, autoraise = True) + if __name__ == "__main__": Onionr() diff --git a/onionr/onionrblacklist.py b/onionr/onionrblacklist.py new file mode 100644 index 00000000..86823283 --- /dev/null +++ b/onionr/onionrblacklist.py @@ -0,0 +1,115 @@ +''' + Onionr - P2P Microblogging Platform & Social network. + + This file handles maintenence of a blacklist database, for blocks and peers +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import sqlite3, os, logger +class OnionrBlackList: + def __init__(self, coreInst): + self.blacklistDB = 'data/blacklist.db' + self._core = coreInst + + if not os.path.exists(self.blacklistDB): + self.generateDB() + return + + def inBlacklist(self, data): + hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data)) + retData = False + if not hashed.isalnum(): + raise Exception("Hashed data is not alpha numeric") + + for i in self._dbExecute("select * from blacklist where hash='%s'" % (hashed,)): + retData = True # this only executes if an entry is present by that hash + break + return retData + + def _dbExecute(self, toExec): + conn = sqlite3.connect(self.blacklistDB) + c = conn.cursor() + retData = c.execute(toExec) + conn.commit() + return retData + + def deleteBeforeDate(self, date): + # TODO, delete blacklist entries before date + return + + def deleteExpired(self, dataType=0): + '''Delete expired entries''' + deleteList = [] + curTime = self._core._utils.getEpoch() + + try: + int(dataType) + except AttributeError: + raise TypeError("dataType must be int") + + for i in self._dbExecute('select * from blacklist where dataType=%s' % (dataType,)): + if i[1] == dataType: + if (curTime - i[2]) >= i[3]: + deleteList.append(i[0]) + + for thing in deleteList: + self._dbExecute("delete from blacklist where hash='%s'" % (thing,)) + + def generateDB(self): + self._dbExecute('''CREATE TABLE blacklist( + hash text primary key not null, + dataType int, + blacklistDate int, + expire int + ); + ''') + return + + def clearDB(self): + self._dbExecute('''delete from blacklist;);''') + + def getList(self): + data = self._dbExecute('select * from blacklist') + myList = [] + for i in data: + myList.append(i[0]) + return myList + + def addToDB(self, data, dataType=0, expire=0): + '''Add to the blacklist. Intended to be block hash, block data, peers, or transport addresses + 0=block + 1=peer + 2=pubkey + ''' + # we hash the data so we can remove data entirely from our node's disk + hashed = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(data)) + + if self.inBlacklist(hashed): + return + + if not hashed.isalnum(): + raise Exception("Hashed data is not alpha numeric") + try: + int(dataType) + except ValueError: + raise Exception("dataType is not int") + try: + int(expire) + except ValueError: + raise Exception("expire is not int") + #TODO check for length sanity + insert = (hashed,) + blacklistDate = self._core._utils.getEpoch() + self._dbExecute("insert into blacklist (hash, dataType, blacklistDate, expire) VALUES('%s', %s, %s, %s);" % (hashed, dataType, blacklistDate, expire)) diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index 695d41ab..e255c0ae 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -28,17 +28,14 @@ class Block: def __init__(self, hash = None, core = None, type = None, content = None): # take from arguments # sometimes people input a bytes object instead of str in `hash` - try: + if (not hash is None) and isinstance(hash, bytes): hash = hash.decode() - except AttributeError: - pass self.hash = hash self.core = core self.btype = type self.bcontent = content - # initialize variables self.valid = True self.raw = None @@ -71,8 +68,10 @@ class Block: # logic - def decrypt(self, anonymous=True, encodedData=True): - '''Decrypt a block, loading decrypted data into their vars''' + def decrypt(self, anonymous = True, encodedData = True): + ''' + Decrypt a block, loading decrypted data into their vars + ''' if self.decrypted: return True retData = False @@ -100,9 +99,11 @@ class Block: else: logger.warn('symmetric decryption is not yet supported by this API') return retData - + def verifySig(self): - '''Verify if a block's signature is signed by its claimed signer''' + ''' + Verify if a block's signature is signed by its claimed signer + ''' core = self.getCore() if core._crypto.edVerify(data=self.signedData, key=self.signer, sig=self.signature, encodedData=True): @@ -227,12 +228,14 @@ class Block: else: self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign) self.update() + return self.getHash() else: logger.warn('Not writing block; it is invalid.') except Exception as e: logger.error('Failed to save block.', error = e, timestamp = False) - return False + + return False # getters @@ -487,7 +490,7 @@ class Block: # static functions - def getBlocks(type = None, signer = None, signed = None, reverse = False, core = None): + def getBlocks(type = None, signer = None, signed = None, parent = None, reverse = False, limit = None, core = None): ''' Returns a list of Block objects based on supplied filters @@ -505,6 +508,9 @@ class Block: try: core = (core if not core is None else onionrcore.Core()) + if (not parent is None) and (not isinstance(parent, Block)): + parent = Block(hash = parent, core = core) + relevant_blocks = list() blocks = (core.getBlockList() if type is None else core.getBlocksByType(type)) @@ -531,9 +537,17 @@ class Block: if not isSigner: relevant = False - if relevant: + if not parent is None: + blockParent = block.getParent() + + if blockParent is None: + relevant = False + else: + relevant = parent.getHash() == blockParent.getHash() + + if relevant and (limit is None or len(relevant_Blocks) <= int(limit)): relevant_blocks.append(block) - + if bool(reverse): relevant_blocks.reverse() diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py new file mode 100644 index 00000000..8410cb80 --- /dev/null +++ b/onionr/onionrdaemontools.py @@ -0,0 +1,56 @@ +''' + Onionr - P2P Microblogging Platform & Social network. + + Contains the CommunicatorUtils class which contains useful functions for the communicator daemon +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import onionrexceptions, onionrpeers, onionrproofs, base64, logger +class DaemonTools: + def __init__(self, daemon): + self.daemon = daemon + self.announceCache = {} + + def announceNode(self): + '''Announce our node to our peers''' + retData = False + + # Announce to random online peers + for i in self.daemon.onlinePeers: + if not i in self.announceCache: + peer = i + break + else: + peer = self.daemon.pickOnlinePeer() + + ourID = self.daemon._core.hsAddress.strip() + + url = 'http://' + peer + '/public/announce/' + data = {'node': ourID} + + combinedNodes = ourID + peer + + if peer in self.announceCache: + data['random'] = self.announceCache[peer] + else: + proof = onionrproofs.DataPOW(combinedNodes, forceDifficulty=4) + data['random'] = base64.b64encode(proof.waitForResult()[1]) + self.announceCache[peer] = data['random'] + + logger.info('Announcing node to ' + url) + if self.daemon._core._utils.doPostRequest(url, data) == 'Success': + retData = True + self.daemon.decrementThreadCount('announceNode') + return retData \ No newline at end of file diff --git a/onionr/onionrevents.py b/onionr/onionrevents.py index 9ecc552f..26fdc093 100644 --- a/onionr/onionrevents.py +++ b/onionr/onionrevents.py @@ -33,10 +33,10 @@ def __event_caller(event_name, data = {}, onionr = None): try: call(plugins.get_plugin(plugin), event_name, data, get_pluginapi(onionr, data)) except ModuleNotFoundError as e: - logger.warn('Disabling nonexistant plugin \"' + plugin + '\"...') + logger.warn('Disabling nonexistant plugin "%s"...' % plugin) plugins.disable(plugin, onionr, stop_event = False) except Exception as e: - logger.warn('Event \"' + event_name + '\" failed for plugin \"' + plugin + '\".') + logger.warn('Event "%s" failed for plugin "%s".' % (event_name, plugin)) logger.debug(str(e)) diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index c97849c1..b26a97d7 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -38,12 +38,23 @@ class InvalidPubkey(Exception): class InvalidMetadata(Exception): pass +class BlacklistedBlock(Exception): + pass + +class DataExists(Exception): + pass + class InvalidHexHash(Exception): '''When a string is not a valid hex string of appropriate length for a hash value''' pass +class InvalidProof(Exception): + '''When a proof is invalid or inadequate''' + pass + # network level exceptions class MissingPort(Exception): pass + class InvalidAddress(Exception): pass diff --git a/onionr/onionrpeers.py b/onionr/onionrpeers.py index b6ed72ec..710f698d 100644 --- a/onionr/onionrpeers.py +++ b/onionr/onionrpeers.py @@ -1,7 +1,7 @@ ''' Onionr - P2P Microblogging Platform & Social network. - This file contains both the OnionrCommunicate class for communcating with peers + This file contains both the PeerProfiles class for network profiling of Onionr nodes ''' ''' This program is free software: you can redistribute it and/or modify @@ -16,4 +16,84 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -''' \ No newline at end of file +''' +import core, config, logger, sqlite3 +class PeerProfiles: + ''' + PeerProfiles + ''' + def __init__(self, address, coreInst): + self.address = address # node address + self.score = None + self.friendSigCount = 0 + self.success = 0 + self.failure = 0 + + if not isinstance(coreInst, core.Core): + raise TypeError("coreInst must be a type of core.Core") + self.coreInst = coreInst + assert isinstance(self.coreInst, core.Core) + + self.loadScore() + return + + def loadScore(self): + '''Load the node's score from the database''' + try: + self.success = int(self.coreInst.getAddressInfo(self.address, 'success')) + except (TypeError, ValueError) as e: + self.success = 0 + self.score = self.success + + def saveScore(self): + '''Save the node's score to the database''' + self.coreInst.setAddressInfo(self.address, 'success', self.score) + return + + def addScore(self, toAdd): + '''Add to the peer's score (can add negative)''' + self.score += toAdd + self.saveScore() + +def getScoreSortedPeerList(coreInst): + if not type(coreInst is core.Core): + raise TypeError('coreInst must be instance of core.Core') + + peerList = coreInst.listAdders() + peerScores = {} + + for address in peerList: + # Load peer's profiles into a list + profile = PeerProfiles(address, coreInst) + peerScores[address] = profile.score + + # Sort peers by their score, greatest to least + peerList = sorted(peerScores, key=peerScores.get, reverse=True) + return peerList + +def peerCleanup(coreInst): + '''Removes peers who have been offline too long or score too low''' + if not type(coreInst is core.Core): + raise TypeError('coreInst must be instance of core.Core') + + logger.info('Cleaning peers...') + config.reload() + + minScore = int(config.get('peers.minimumScore')) + maxPeers = int(config.get('peers.maxStoredPeers')) + + adders = getScoreSortedPeerList(coreInst) + adders.reverse() + + for address in adders: + # Remove peers that go below the negative score + if PeerProfiles(address, coreInst).score < minScore: + coreInst.removeAddress(address) + try: + coreInst._blacklist.addToDB(address, dataType=1, expire=300) + except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue + pass + logger.warn('Removed address ' + address + '.') + + # Unban probably not malicious peers TODO improve + coreInst._blacklist.deleteExpired(dataType=1) \ No newline at end of file diff --git a/onionr/onionrpluginapi.py b/onionr/onionrpluginapi.py index bfaf73e8..0120dad7 100644 --- a/onionr/onionrpluginapi.py +++ b/onionr/onionrpluginapi.py @@ -130,6 +130,22 @@ class CommandAPI: def get_commands(self): return self.pluginapi.get_onionr().getCommands() +class WebAPI: + def __init__(self, pluginapi): + self.pluginapi = pluginapi + + def register_callback(self, action, callback, scope = 'public'): + return self.pluginapi.get_onionr().api.setCallback(action, callback, scope = scope) + + def unregister_callback(self, action, scope = 'public'): + return self.pluginapi.get_onionr().api.removeCallback(action, scope = scope) + + def get_callback(self, action, scope = 'public'): + return self.pluginapi.get_onionr().api.getCallback(action, scope= scope) + + def get_callbacks(self, scope = None): + return self.pluginapi.get_onionr().api.getCallbacks(scope = scope) + class pluginapi: def __init__(self, onionr, data): self.onionr = onionr @@ -142,6 +158,7 @@ class pluginapi: self.daemon = DaemonAPI(self) self.plugins = PluginAPI(self) self.commands = CommandAPI(self) + self.web = WebAPI(self) def get_onionr(self): return self.onionr @@ -167,5 +184,8 @@ class pluginapi: def get_commandapi(self): return self.commands + def get_webapi(self): + return self.web + def is_development_mode(self): return self.get_onionr()._developmentMode diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index 194b37e9..b93d5724 100644 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -22,16 +22,19 @@ import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, lo import core class DataPOW: - def __init__(self, data, threadCount = 5): + def __init__(self, data, forceDifficulty=0, threadCount = 5): self.foundHash = False self.difficulty = 0 self.data = data self.threadCount = threadCount - dataLen = sys.getsizeof(data) - self.difficulty = math.floor(dataLen / 1000000) - if self.difficulty <= 2: - self.difficulty = 4 + if forceDifficulty == 0: + dataLen = sys.getsizeof(data) + self.difficulty = math.floor(dataLen / 1000000) + if self.difficulty <= 2: + self.difficulty = 4 + else: + self.difficulty = forceDifficulty try: self.data = self.data.encode() diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 33e3f51d..7e0abd94 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -52,23 +52,16 @@ class OnionrUtils: with open('data/time-bypass.txt', 'r') as bypass: self.timingToken = bypass.read() except Exception as error: - logger.error('Failed to fetch time bypass token.', error=error) + logger.error('Failed to fetch time bypass token.', error = error) - def sendPM(self, pubkey, message): + return self.timingToken + + def getRoundedEpoch(self, roundS=60): ''' - High level function to encrypt a message to a peer and insert it as a block - ''' - - self._core.insertBlock(message, header='pm', sign=True, encryptType='asym', asymPeer=pubkey) - - return - - def getCurrentHourEpoch(self): - ''' - Returns the current epoch, rounded down to the hour + Returns the epoch, rounded down to given seconds (Default 60) ''' epoch = self.getEpoch() - return epoch - (epoch % 3600) + return epoch - (epoch % roundS) def incrementAddressSuccess(self, address): ''' @@ -133,7 +126,8 @@ class OnionrUtils: retVal = False if newAdderList != False: for adder in newAdderList.split(','): - if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress(): + adder = adder.strip() + if not adder in self._core.listAdders(randomOrder = False) and adder != self.getMyAddress() and not self._core._blacklist.inBlacklist(adder): if self._core.addAddress(adder): logger.info('Added %s to db.' % adder, timestamp = True) retVal = True @@ -207,22 +201,29 @@ class OnionrUtils: def getBlockMetadataFromData(self, blockData): ''' accepts block contents as string, returns a tuple of metadata, meta (meta being internal metadata, which will be returned as an encrypted base64 string if it is encrypted, dict if not). - + ''' meta = {} + metadata = {} + data = blockData try: blockData = blockData.encode() except AttributeError: pass - metadata = json.loads(blockData[:blockData.find(b'\n')].decode()) - data = blockData[blockData.find(b'\n'):].decode() - if not metadata['encryptType'] in ('asym', 'sym'): - try: - meta = json.loads(metadata['meta']) - except KeyError: - pass - meta = metadata['meta'] + try: + metadata = json.loads(blockData[:blockData.find(b'\n')].decode()) + except json.decoder.JSONDecodeError: + pass + else: + data = blockData[blockData.find(b'\n'):].decode() + + if not metadata['encryptType'] in ('asym', 'sym'): + try: + meta = json.loads(metadata['meta']) + except KeyError: + pass + meta = metadata['meta'] return (metadata, meta, data) def checkPort(self, port, host=''): @@ -252,7 +253,7 @@ class OnionrUtils: return False else: return True - + def processBlockMetadata(self, blockHash): ''' Read metadata from a block and cache it to the block database @@ -270,7 +271,7 @@ class OnionrUtils: def escapeAnsi(self, line): ''' Remove ANSI escape codes from a string with regex - + taken or adapted from: https://stackoverflow.com/a/38662876 ''' ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]') @@ -332,12 +333,12 @@ class OnionrUtils: retVal = False return retVal - - def validateMetadata(self, metadata): + + def validateMetadata(self, metadata, blockData): '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string''' # TODO, make this check sane sizes retData = False - + # convert to dict if it is json string if type(metadata) is str: try: @@ -363,7 +364,24 @@ class OnionrUtils: break else: # if metadata loop gets no errors, it does not break, therefore metadata is valid - retData = True + # make sure we do not have another block with the same data content (prevent data duplication and replay attacks) + nonce = self._core._utils.bytesToStr(self._core._crypto.sha3Hash(blockData)) + try: + with open(self._core.dataNonceFile, 'r') as nonceFile: + if nonce in nonceFile.read(): + retData = False # we've seen that nonce before, so we can't pass metadata + raise onionrexceptions.DataExists + except FileNotFoundError: + retData = True + except onionrexceptions.DataExists: + # do not set retData to True, because nonce has been seen before + pass + else: + retData = True + if retData: + # Executes if data not seen + with open(self._core.dataNonceFile, 'a') as nonceFile: + nonceFile.write(nonce + '\n') else: logger.warn('In call to utils.validateMetadata, metadata must be JSON string or a dictionary object') @@ -383,7 +401,7 @@ class OnionrUtils: else: retVal = True return retVal - + def isIntegerString(self, data): '''Check if a string is a valid base10 integer''' try: @@ -525,22 +543,22 @@ class OnionrUtils: '''returns epoch''' return math.floor(time.time()) - def doGetRequest(self, url, port=0, proxyType='tor'): + def doPostRequest(self, url, data={}, port=0, proxyType='tor'): ''' - Do a get request through a local tor or i2p instance + Do a POST request through a local tor or i2p instance ''' if proxyType == 'tor': if port == 0: - raise onionrexceptions.MissingPort('Socks port required for Tor HTTP get request') - proxies = {'http': 'socks5://127.0.0.1:' + str(port), 'https': 'socks5://127.0.0.1:' + str(port)} + port = self._core.torPort + proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)} elif proxyType == 'i2p': proxies = {'http': 'http://127.0.0.1:4444'} else: return headers = {'user-agent': 'PyOnionr'} try: - proxies = {'http': 'socks5h://127.0.0.1:' + str(port), 'https': 'socks5h://127.0.0.1:' + str(port)} - r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30)) + proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)} + r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30)) retData = r.text except KeyboardInterrupt: raise KeyboardInterrupt @@ -549,7 +567,34 @@ class OnionrUtils: retData = False return retData - def getNistBeaconSalt(self, torPort=0): + def doGetRequest(self, url, port=0, proxyType='tor'): + ''' + Do a get request through a local tor or i2p instance + ''' + retData = False + if proxyType == 'tor': + if port == 0: + raise onionrexceptions.MissingPort('Socks port required for Tor HTTP get request') + proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)} + elif proxyType == 'i2p': + proxies = {'http': 'http://127.0.0.1:4444'} + else: + return + headers = {'user-agent': 'PyOnionr'} + try: + proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)} + r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30)) + retData = r.text + except KeyboardInterrupt: + raise KeyboardInterrupt + except ValueError as e: + logger.debug('Failed to make request', error = e) + except requests.exceptions.RequestException as e: + logger.debug('Error: %s' % str(e)) + retData = False + return retData + + def getNistBeaconSalt(self, torPort=0, rounding=3600): ''' Get the token for the current hour from the NIST randomness beacon ''' @@ -559,7 +604,7 @@ class OnionrUtils: except IndexError: raise onionrexceptions.MissingPort('Missing Tor socks port') retData = '' - curTime = self._core._utils.getCurrentHourEpoch + curTime = self.getRoundedEpoch(rounding) self.nistSaltTimestamp = curTime data = self.doGetRequest('https://beacon.nist.gov/rest/record/' + str(curTime), port=torPort) dataXML = minidom.parseString(data, forbid_dtd=True, forbid_entities=True, forbid_external=True) @@ -570,6 +615,19 @@ class OnionrUtils: else: self.powSalt = retData return retData + + def strToBytes(self, data): + try: + data = data.encode() + except AttributeError: + pass + return data + def bytesToStr(self, data): + try: + data = data.decode() + except AttributeError: + pass + return data def size(path='.'): ''' diff --git a/onionr/static-data/connect-check.txt b/onionr/static-data/connect-check.txt new file mode 100644 index 00000000..009a2a9a --- /dev/null +++ b/onionr/static-data/connect-check.txt @@ -0,0 +1 @@ +https://3g2upl4pq6kufc4m.onion/robots.txt,http://expyuzz4wqqyqhjn.onion/robots.txt,https://onionr.voidnet.tech/ diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index 3da4d268..300c9625 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -22,6 +22,8 @@ import logger, config, threading, time, readline, datetime from onionrblockapi import Block import onionrexceptions +import locale +locale.setlocale(locale.LC_ALL, '') plugin_name = 'pms' PLUGIN_VERSION = '0.0.1' @@ -107,14 +109,13 @@ class OnionrMail: pass else: readBlock.verifySig() - print('Message recieved from', readBlock.signer) + print('Message recieved from %s' % (readBlock.signer,)) print('Valid signature:', readBlock.validSig) if not readBlock.validSig: - logger.warn('This message has an INVALID signature. Anyone could have sent this message.') - logger.readline('Press enter to continue to message.') - - print(draw_border(self.myCore._utils.escapeAnsi(readBlock.bcontent.decode().strip()))) - + logger.warn('This message has an INVALID signature. ANYONE could have sent this message.') + cancel = logger.readline('Press enter to continue to message, or -q to not open the message (recommended).') + if cancel != '-q': + print(draw_border(self.myCore._utils.escapeAnsi(readBlock.bcontent.decode().strip()))) return def draftMessage(self): diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 9188aa66..fcbf733e 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -2,8 +2,26 @@ "general" : { "dev_mode": true, "display_header" : true, - "dc_response": true, - "dc_execcallbacks" : true + + "direct_connect" : { + "respond" : true, + "execute_callbacks" : true + } + }, + + "www" : { + "public" : { + "run" : true + }, + + "private" : { + "run" : true + }, + + "ui" : { + "run" : true, + "private" : true + } }, "client" : { @@ -33,9 +51,14 @@ }, "allocations":{ - "disk": 1000000000, + "disk": 9000000000, "netTotal": 1000000000, - "blockCache" : 5000000, - "blockCacheTotal" : 50000000 + "blockCache": 5000000, + "blockCacheTotal": 50000000 + }, + "peers":{ + "minimumScore": -100, + "maxStoredPeers": 500, + "maxConnect": 5 } } diff --git a/onionr/static-data/index.html b/onionr/static-data/index.html index f9df9eb7..93e48beb 100644 --- a/onionr/static-data/index.html +++ b/onionr/static-data/index.html @@ -1,5 +1,7 @@

This is an Onionr Node

-

The content on this server is not necessarily created or intentionally stored by the owner of the server.

+

The content on this server is not necessarily created by the server owner, and was not necessarily stored with the owner's knowledge.

+ +

Onionr is a decentralized, distributed data storage system, that anyone can insert data into.

To learn more about Onionr, see the website at https://Onionr.VoidNet.tech/

diff --git a/onionr/static-data/www/ui/README.md b/onionr/static-data/www/ui/README.md new file mode 100644 index 00000000..451b08ed --- /dev/null +++ b/onionr/static-data/www/ui/README.md @@ -0,0 +1,44 @@ +# Onionr UI + +## About + +The default GUI for Onionr + +## Setup + +To compile the application, simply execute the following: + +``` +python3 compile.py +``` + +If you are wanting to compile Onionr UI for another language, execute the following, replacing `[lang]` with the target language (supported languages include `eng` for English, `spa` para español, and `zho`为中国人): + +``` +python3 compile.py [lang] +``` + +## FAQ +### Why "compile" anyway? +This web application is compiled for a few reasons: +1. To make it easier to update; this way, we do not have to update the header in every file if we want to change something about it. +2. To make the application smaller in size; there is less duplicated code when the code like the header and footer can be stored in an individual file rather than every file. +3. For multi-language support; with the Python "tags" feature, we can reference strings by variable name, and based on a language file, they can be dynamically inserted into the page on compilation. +4. For compile-time customizations. + +### What exactly happens when you compile? +Upon compilation, files from the `src/` directory will be copied to `dist/` directory, header and footers will be injected in the proper places, and Python "tags" will be interpreted. + + +### How do Python "tags" work? +There are two types of Python "tags": +1. Logic tags (`<$ logic $>`): These tags allow you to perform logic at compile time. Example: `<$ import datetime; lastUpdate = datetime.datetime.now() $>`: This gets the current time while compiling, then stores it in `lastUpdate`. +2. Data tags (`<$= data $>`): These tags take whatever the return value of the statement in the tags is, and write it directly to the page. Example: `<$= 'This application was compiled at %s.' % lastUpdate $>`: This will write the message in the string in the tags to the page. + +**Note:** Logic tags take a higher priority and will always be interpreted first. + +### How does the language feature work? +When you use a data tag to write a string to the page (e.g. `<$= LANG.HELLO_WORLD $>`), the language feature simply takes dictionary of the language that is currently being used from the language map file (`lang.json`), then searches for the key (being the variable name after the characters `LANG.` in the data tag, like `HELLO_WORLD` from the example before). It then writes that string to the page. Language variables are always prefixed with `LANG.` and should always be uppercase (as they are a constant). + +### I changed a few things in the application and tried to view the updates in my browser, but nothing changed! +You most likely forgot to compile. Try running `python3 compile.py` and check again. If you are still having issues, [open up an issue](https://gitlab.com/beardog/Onionr/issues/new?issue[title]=Onionr UI not updating after compiling). \ No newline at end of file diff --git a/onionr/static-data/www/ui/common/footer.html b/onionr/static-data/www/ui/common/footer.html new file mode 100644 index 00000000..6b5cfb06 --- /dev/null +++ b/onionr/static-data/www/ui/common/footer.html @@ -0,0 +1,4 @@ + + + + diff --git a/onionr/static-data/www/ui/common/header.html b/onionr/static-data/www/ui/common/header.html new file mode 100644 index 00000000..2a2b4f56 --- /dev/null +++ b/onionr/static-data/www/ui/common/header.html @@ -0,0 +1,30 @@ +<$= LANG.ONIONR_TITLE $> + + + + + + + + + + diff --git a/onionr/static-data/www/ui/common/onionr-timeline-post.html b/onionr/static-data/www/ui/common/onionr-timeline-post.html new file mode 100644 index 00000000..68440a01 --- /dev/null +++ b/onionr/static-data/www/ui/common/onionr-timeline-post.html @@ -0,0 +1,32 @@ + +
+
+
+
+ +
+
+
+
+ $user-name + $user-id-truncated +
+ +
+
$date-relative
+
+
+ +
+ $content +
+ +
+ <$= LANG.POST_LIKE $> + <$= LANG.POST_REPLY $> +
+
+
+
+
+ diff --git a/onionr/static-data/www/ui/compile.py b/onionr/static-data/www/ui/compile.py new file mode 100755 index 00000000..2667b210 --- /dev/null +++ b/onionr/static-data/www/ui/compile.py @@ -0,0 +1,130 @@ +#!/usr/bin/python3 + +import shutil, os, re, json, traceback + +# get user's config +settings = {} +with open('config.json', 'r') as file: + settings = json.loads(file.read()) + +# "hardcoded" config, not for user to mess with +HEADER_FILE = 'common/header.html' +FOOTER_FILE = 'common/footer.html' +SRC_DIR = 'src/' +DST_DIR = 'dist/' +HEADER_STRING = '
' +FOOTER_STRING = '