diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 292dfb14..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,6 +0,0 @@ -test: - script: - - apt-get update -qy - - apt-get install -y python3-dev python3-pip tor - - pip3 install -r requirements.txt - - make test \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 603021b5..00000000 --- a/.travis.yml +++ /dev/null @@ -1,8 +0,0 @@ -language: python -python: - - "3.6.4" -# install dependencies -install: - - sudo apt install tor - - pip install -r requirements.txt -script: make test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 91a0e227..f5a31c5d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,7 +27,7 @@ And most importantly, please be patient. Onionr is an open source project done b ## Asking Questions -If you need help with Onionr, you can ask in our +If you need help with Onionr, you can contact the devs (be polite and remember this is a volunteer-driven non-profit project). ## Contributing Code diff --git a/Makefile b/Makefile index 7e235b4f..c3616e3d 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ uninstall: rm -f $(DESTDIR)$(PREFIX)/bin/onionr test: - @./run-linux stop + @./onionr.sh stop @sleep 1 @rm -rf onionr/data-backup @mv onionr/data onionr/data-backup | true > /dev/null 2>&1 @@ -29,15 +29,15 @@ test: soft-reset: @echo "Soft-resetting Onionr..." rm -f onionr/data/blocks/*.dat onionr/data/*.db onionr/data/block-nonces.dat | true > /dev/null 2>&1 - @./run-linux version | grep -v "Failed" --color=always + @./onionr.sh version | grep -v "Failed" --color=always reset: @echo "Hard-resetting Onionr..." rm -rf onionr/data/ | true > /dev/null 2>&1 cd onionr/static-data/www/ui/; rm -rf ./dist; python compile.py - #@./RUN-LINUX.sh version | grep -v "Failed" --color=always + #@./onionr.sh.sh version | grep -v "Failed" --color=always plugins-reset: @echo "Resetting plugins..." rm -rf onionr/data/plugins/ | true > /dev/null 2>&1 - @./run-linux version | grep -v "Failed" --color=always + @./onionr.sh version | grep -v "Failed" --color=always diff --git a/README.md b/README.md index 5d1aa37b..5a346e5d 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,71 @@ -![Onionr logo](./docs/onionr-logo.png) +

-(***experimental, not safe or easy to use yet***) + + +

+ +

+ Anonymous P2P storage network 🕵️ +

+ +(***pre-alpha & experimental, not well tested or easy to use yet***) [![Open Source Love](https://badges.frapsoft.com/os/v3/open-source.png?v=103)](https://github.com/ellerbrock/open-source-badges/) - -Anonymous P2P platform, using Tor & I2P. -
-**The main repo for this software is at https://gitlab.com/beardog/Onionr/** - +**The main repository for this software is at https://gitlab.com/beardog/Onionr/** # Summary -Onionr is a decentralized, peer-to-peer data storage network, designed to be anonymous and resistant to (meta)data analysis and spam. +Onionr is a decentralized, peer-to-peer data storage network, designed to be anonymous and resistant to (meta)data analysis and spam/disruption. + +Onionr stores data in independent packages referred to as 'blocks'. The blocks are synced to all other nodes in the network. Blocks and user IDs cannot be easily proven to have been created by particular nodes (only inferred). Even if there is enough evidence to believe a particular node created a block, nodes still operate behind Tor or I2P and as such are not trivially known to be at a particular IP address. + +Users are identified by ed25519 public keys, which can be used to sign blocks or send encrypted data. Onionr can be used for mail, as a social network, instant messenger, file sharing software, or for encrypted group discussion. -# Roadmap/features +![Tor stinks slide image](docs/tor-stinks-02.png) -Check the [Gitlab Project](https://gitlab.com/beardog/Onionr/milestones/1) to see progress towards the alpha release. - -## Core internal features +## Main Features * [X] Fully p2p/decentralized, no trackers or other single points of failure * [X] End to end encryption of user data * [X] Optional non-encrypted blocks, useful for blog posts or public file sharing * [X] Easy API system for integration to websites -* [ ] Metadata analysis resistance (being improved) - - -## Other features +* [X] Metadata analysis resistance +* [X] Transport agnosticism (no internet required) **Onionr API and functionality is subject to non-backwards compatible change during pre-alpha development** +# Install and Run on Linux + +The following applies to Ubuntu Bionic. Other distros may have different package or command names. + +* Have python3.5+, python3-pip, Tor (daemon, not browser) installed (python3-dev recommended) +* Clone the git repo: `$ git clone https://gitlab.com/beardog/onionr` +* cd into install direction: `$ cd onionr/` +* Install the Python dependencies ([virtualenv strongly recommended](https://virtualenv.pypa.io/en/stable/userguide/)): `$ pip3 install -r requirements.txt` + ## Help out -Everyone is welcome to help out. Please get in touch first if you are making non-trivial changes. If you can't help with programming, you can write documentation or guides. +Everyone is welcome to help out. Help is wanted for the following: -Bitcoin/Bitcoin Cash: 1onion55FXzm6h8KQw3zFw2igpHcV7LPq +* Development (Get in touch first) + * Creation of a lib for use from other languages and faster proof-of-work + * Android and IOS development + * Windows and Mac support + * General bug fixes and development of new features +* Testing +* Running stable nodes +* Security review/audit + +Bitcoin: [1onion55FXzm6h8KQw3zFw2igpHcV7LPq](bitcoin:1onion55FXzm6h8KQw3zFw2igpHcV7LPq) +USD: [Ko-Fi](https://www.ko-fi.com/beardogkf) ## Disclaimer The Tor Project, I2P developers, and anyone else do not own, create, or endorse this project, and are not otherwise involved. -The badges (besides travis-ci build) are by Maik Ellerbrock is licensed under a Creative Commons Attribution 4.0 International License. +The 'open source badge' is by Maik Ellerbrock and is licensed under a Creative Commons Attribution 4.0 International License. \ No newline at end of file diff --git a/docs/onionr-logo.png b/docs/onionr-logo.png index b6c3c9b5..31a95241 100644 Binary files a/docs/onionr-logo.png and b/docs/onionr-logo.png differ diff --git a/docs/onionr-logo.png~ b/docs/onionr-logo.png~ new file mode 100644 index 00000000..5e15d42f Binary files /dev/null and b/docs/onionr-logo.png~ differ diff --git a/docs/tor-stinks-02.png b/docs/tor-stinks-02.png new file mode 100644 index 00000000..ca35b23c Binary files /dev/null and b/docs/tor-stinks-02.png differ diff --git a/docs/whitepaper.md b/docs/whitepaper.md index e791b83c..8c5d0f96 100644 --- a/docs/whitepaper.md +++ b/docs/whitepaper.md @@ -5,7 +5,7 @@ # Introduction -The most important thing in the modern world is information. The ability to communicate freely with others. The internet has provided humanity with the ability to spread information globally, but there are many people who try (and sometimes succeed) to stifle the flow of information. +One of the most important things in the modern world is information. The ability to communicate freely with others is crucial for maintaining personal liberties. The internet has provided humanity with the ability to spread information globally, but there are many people who try (and sometimes succeed) to stifle the flow of information. Internet censorship comes in many forms, state censorship, corporate consolidation of media, threats of violence, network exploitation (e.g. denial of service attacks). @@ -14,25 +14,22 @@ To prevent censorship or loss of information, these measures must be in place: * Resistance to censorship of underlying infrastructure or of network hosts * Anonymization of users by default - * The Inability to violently coerce human users (personal threats/"doxxing", or totalitarian regime censorship) + * The Inability to coerce human users (personal threats/"doxxing", or totalitarian regime censorship) -* Economic availability. A system should not rely on a single device to be constantly online, and should not be overly expensive to use. The majority of people in the world own cell phones, but comparatively few own personal computers, particularly in developing countries. +* Economic availability. A system should not rely on a single device to be constantly online, and should not be overly expensive to use. The majority of people in the world own cell phones, but comparatively few own personal computers, particularly in developing countries. Internet connectivity can be slow or spotty in many areas. There are many great projects that tackle decentralization and privacy issues, but there are none which tackle all of the above issues. Some of the existing networks have also not worked well in practice, or are more complicated than they need to be. # Onionr Design Goals -When designing Onionr we had these goals in mind: +When designing Onionr we had these main goals in mind: * Anonymous Blocks - - * Difficult to determine block creator or users regardless of transport used -* Default Anonymous Transport Layer - * Tor and I2P + * Difficult to determine block creator or users regardless of transport used +* Node-anonymity * Transport agnosticism -* Default global sync, but can configure what blocks to seed +* Default global sync, but configurable * Spam resistance -* Encrypted blocks # Onionr Design @@ -40,23 +37,23 @@ When designing Onionr we had these goals in mind: ## General Overview -At its core, Onionr is merely a description for storing data in self-verifying packages ("blocks"). These blocks can be encrypted to a user (or self), encrypted symmetrically, or not at all. Blocks can be signed by their creator, but regardless, they are self-verifying due to being identified by a sha3-256 hash value; once a block is created, it cannot be modified. +At its core, Onionr is merely a description for storing data in self-verifying packages ("blocks"). These blocks can be encrypted to a user (or for one's self), encrypted symmetrically, or not at all. Blocks can be signed by their creator, but regardless, they are self-verifying due to being identified by a sha3-256 hash value; once a block is created, it cannot be modified. -Onionr exchanges a list of blocks between all nodes. By default, all nodes download and share all other blocks, however this is configurable. +Onionr exchanges a list of blocks between all nodes. By default, all nodes download and share all other blocks, however this is configurable. Blocks do not rely on any particular order of receipt or transport mechanism. ## User IDs User IDs are simply Ed25519 public keys. They are represented in Base32 format, or encoded using the [PGP Word List](https://en.wikipedia.org/wiki/PGP_word_list). -Public keys can be generated deterministicly with a password using a key derivation function (Argon2id). This password can be shared between many users in order to share data anonymously among a group, using only 1 password. This is useful in some cases, but is risky, as if one user causes the key to be compromised and does not notify the group or revoke the key, there is no way to know. +Public keys can be generated deterministically with a password using a key derivation function (Argon2id). This password can be shared between many users in order to share data anonymously among a group, using only 1 password. This is useful in some cases, but is risky, as if one user causes the key to be compromised and does not notify the group or revoke the key, there is no way to know. ## Nodes -Although Onionr is transport agnostic, the only supported transports in the reference implemetation are Tor .onion services and I2P hidden services. Nodes announce their address on creation. +Although Onionr is transport agnostic, the only supported transports in the reference implementation are Tor .onion services and I2P hidden services. Nodes announce their address on creation by connecting to peers specified in a bootstrap file. Peers in the bootstrap file have no special permissions aside from being default peers. ### Node Profiling -To mitigate maliciously slow or unreliable nodes, Onionr builds a profile on nodes it connects to. Nodes are assigned a score, which raises based on the amount of successful block transfers, speed, and reliabilty of a node, and reduces based on how unreliable a node is. If a node is unreachable for over 24 hours after contact, it is forgotten. Onionr can also prioritize connection to 'friend' nodes. +To mitigate maliciously slow or unreliable nodes, Onionr builds a profile on nodes it connects to. Nodes are assigned a score, which raises based on the amount of successful block transfers, speed, and reliability of a node, and reduces the score based on how unreliable a node is. If a node is unreachable for over 24 hours after contact, it is forgotten. Onionr can also prioritize connection to 'friend' nodes. ## Block Format @@ -90,8 +87,10 @@ Onionr can provide evidence of when a block was inserted by requesting other use This can be done either by the creator of the block prior to generation, or by any node after insertion. -In addition, randomness beacons such as the one operated by [NIST](https://beacon.nist.gov/home) or the hash of the latest blocks in a cryptocurrency network could be used to affirm that a block was at least not *created* before a given time. +In addition, randomness beacons such as the one operated by [NIST](https://beacon.nist.gov/home), [Chile](https://beacon.clcert.cl/), or the hash of the latest blocks in a cryptocurrency network could be used to affirm that a block was at least not *created* before a given time. # Direct Connections -We propose a system to \ No newline at end of file +We propose a method of using Onionr's block sync system to enable direct connections between peers by having one peer request to connect to another using the peer's public key. Since the request is within a standard block, proof of work must be used to request connection. If the requested peer is available and wishes to accept the connection,Onionr will generate a temporary .onion address for the other peer to connect to. Alternatively, a reverse connection may be formed, which is faster to establish but requires a message brokering system instead of a standard socket. + +The benefits of such a system are increased privacy, and the ability to anonymously communicate from multiple devices at once. In a traditional onion service, one's online status can be monitored and more easily correlated. \ No newline at end of file diff --git a/run-linux b/onionr.sh similarity index 100% rename from run-linux rename to onionr.sh diff --git a/onionr/api.py b/onionr/api.py index f983677d..57f34ac0 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -17,14 +17,28 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import flask +from gevent.pywsgi import WSGIServer, WSGIHandler +from gevent import Timeout +#import gevent.monkey +#gevent.monkey.patch_socket() +import flask, cgi from flask import request, Response, abort, send_from_directory -from gevent.pywsgi import WSGIServer -import sys, random, threading, hmac, hashlib, base64, time, math, os, json +import sys, random, threading, hmac, hashlib, base64, time, math, os, json, socket import core from onionrblockapi import Block import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionr +class FDSafeHandler(WSGIHandler): + def handle(self): + timeout = Timeout(60, exception=Exception) + timeout.start() + + #timeout = gevent.Timeout.start_new(3) + try: + WSGIHandler.handle(self) + except Timeout as ex: + raise + def guessMime(path): ''' Guesses the mime type of a file from the input filename @@ -47,9 +61,19 @@ def setBindIP(filePath): '''Set a random localhost IP to a specified file (intended for private or public API localhost IPs)''' hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))] data = '.'.join(hostOctets) - + + # Try to bind IP. Some platforms like Mac block non normal 127.x.x.x + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.bind((data, 0)) + except OSError: + logger.warn('Your platform appears to not support random local host addresses 127.x.x.x. Falling back to 127.0.0.1.') + data = '127.0.0.1' + s.close() + with open(filePath, 'w') as bindFile: bindFile.write(data) + return data class PublicAPI: @@ -70,6 +94,9 @@ class PublicAPI: @app.before_request def validateRequest(): '''Validate request has the correct hostname''' + # If high security level, deny requests to public + if config.get('general.security_level', default=0) > 0: + abort(403) if type(self.torAdder) is None and type(self.i2pAdder) is None: # abort if our hs addresses are not known abort(403) @@ -84,6 +111,7 @@ class PublicAPI: resp.headers['X-Frame-Options'] = 'deny' resp.headers['X-Content-Type-Options'] = "nosniff" resp.headers['X-API'] = onionr.API_VERSION + resp.headers['Connection'] = "close" return resp @app.route('/') @@ -97,7 +125,8 @@ class PublicAPI: @app.route('/getblocklist') def getBlockList(): - bList = clientAPI._core.getBlockList() + dateAdjust = request.args.get('date') + bList = clientAPI._core.getBlockList(dateRec=dateAdjust) for b in self.hideBlocks: if b in bList: bList.remove(b) @@ -109,9 +138,9 @@ class PublicAPI: data = name if clientAPI._utils.validateHash(data): if data not in self.hideBlocks: - if os.path.exists(clientAPI._core.dataDir + 'blocks/' + data + '.dat'): - block = Block(hash=data.encode(), core=clientAPI._core) - resp = base64.b64encode(block.getRaw().encode()).decode() + if data in clientAPI._core.getBlockList(): + block = clientAPI.getBlockData(data, raw=True).encode() + resp = base64.b64encode(block).decode() if len(resp) == 0: abort(404) resp = "" @@ -133,9 +162,9 @@ class PublicAPI: @app.route('/pex') def peerExchange(): - response = ','.join(clientAPI._core.listAdders()) + response = ','.join(clientAPI._core.listAdders(recent=3600)) if len(response) == 0: - response = 'none' + response = '' return Response(response) @app.route('/announce', methods=['post']) @@ -204,7 +233,7 @@ class PublicAPI: clientAPI._core.refreshFirstStartVars() self.torAdder = clientAPI._core.hsAddress time.sleep(1) - self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None) + self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None, handler_class=FDSafeHandler) self.httpServer.serve_forever() class API: @@ -221,19 +250,23 @@ class API: This initilization defines all of the API entry points and handlers for the endpoints and errors This also saves the used host (random localhost IP address) to the data folder in host.txt ''' - + # assert isinstance(onionrInst, onionr.Onionr) # configure logger and stuff onionr.Onionr.setupConfig('data/', self = self) self.debug = debug self._privateDelayTime = 3 - self._core = core.Core() + self._core = onionrInst.onionrCore + self.startTime = self._core._utils.getEpoch() self._crypto = onionrcrypto.OnionrCrypto(self._core) self._utils = onionrutils.OnionrUtils(self._core) app = flask.Flask(__name__) bindPort = int(config.get('client.client.port', 59496)) self.bindPort = bindPort + # Be extremely mindful of this + self.whitelistEndpoints = ('site', 'www', 'onionrhome', 'board', 'boardContent', 'sharedContent', 'mail', 'mailindex') + self.clientToken = config.get('client.webpassword') self.timeBypassToken = base64.b16encode(os.urandom(32)).decode() @@ -242,6 +275,8 @@ class API: self.host = setBindIP(self._core.privateApiHostFile) logger.info('Running api on %s:%s' % (self.host, self.bindPort)) self.httpServer = '' + + self.queueResponse = {} onionrInst.setClientAPIInst(self) @app.before_request @@ -249,6 +284,8 @@ class API: '''Validate request has set password and is the correct hostname''' if request.host != '%s:%s' % (self.host, self.bindPort): abort(403) + if request.endpoint in self.whitelistEndpoints: + return try: if not hmac.compare_digest(request.headers['token'], self.clientToken): abort(403) @@ -257,28 +294,105 @@ class API: @app.after_request def afterReq(resp): - resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" + #resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" + resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self'; img-src 'self'; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'self'" resp.headers['X-Frame-Options'] = 'deny' resp.headers['X-Content-Type-Options'] = "nosniff" resp.headers['X-API'] = onionr.API_VERSION resp.headers['Server'] = '' resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch. + resp.headers['Connection'] = "close" return resp + @app.route('/board/', endpoint='board') + def loadBoard(): + return send_from_directory('static-data/www/board/', "index.html") + + @app.route('/mail/', endpoint='mail') + def loadMail(path): + return send_from_directory('static-data/www/mail/', path) + @app.route('/mail/', endpoint='mailindex') + def loadMailIndex(): + return send_from_directory('static-data/www/mail/', 'index.html') + + @app.route('/board/', endpoint='boardContent') + def boardContent(path): + return send_from_directory('static-data/www/board/', path) + @app.route('/shared/', endpoint='sharedContent') + def sharedContent(path): + return send_from_directory('static-data/www/shared/', path) + + @app.route('/www/', endpoint='www') + def wwwPublic(path): + if not config.get("www.private.run", True): + abort(403) + return send_from_directory(config.get('www.private.path', 'static-data/www/private/'), path) + + @app.route('/queueResponseAdd/', methods=['post']) + def queueResponseAdd(name): + self.queueResponse[name] = request.form['data'] + return Response('success') + + @app.route('/queueResponse/') + def queueResponse(name): + resp = 'failure' + try: + resp = self.queueResponse[name] + except KeyError: + pass + else: + del self.queueResponse[name] + return Response(resp) + @app.route('/ping') def ping(): return Response("pong!") - @app.route('/') + @app.route('/', endpoint='onionrhome') def hello(): - return Response("hello client") + return send_from_directory('static-data/www/private/', 'index.html') + + @app.route('/getblocksbytype/') + def getBlocksByType(name): + blocks = self._core.getBlocksByType(name) + return Response(','.join(blocks)) + + @app.route('/gethtmlsafeblockdata/') + def getSafeData(name): + resp = '' + if self._core._utils.validateHash(name): + try: + resp = cgi.escape(Block(name).bcontent, quote=True) + except TypeError: + pass + else: + abort(404) + return Response(resp) + + @app.route('/getblockdata/') + def getData(name): + resp = "" + if self._core._utils.validateHash(name): + if name in self._core.getBlockList(): + try: + resp = self.getBlockData(name, decrypt=True) + except ValueError: + pass + else: + abort(404) + else: + abort(404) + return Response(resp) - @app.route('/site/') - def site(): - bHash = block + @app.route('/site/', endpoint='site') + def site(name): + bHash = name resp = 'Not Found' if self._core._utils.validateHash(bHash): - resp = Block(bHash).bcontent + try: + resp = Block(bHash).bcontent + except TypeError: + pass try: resp = base64.b64decode(resp) except: @@ -306,7 +420,26 @@ class API: pass return Response("bye") - self.httpServer = WSGIServer((self.host, bindPort), app, log=None) + @app.route('/shutdownclean') + def shutdownClean(): + # good for calling from other clients + self._core.daemonQueueAdd('shutdown') + return Response("bye") + + @app.route('/getstats') + def getStats(): + #return Response("disabled") + while True: + try: + return Response(self._core.serializer.getStats()) + except AttributeError: + pass + + @app.route('/getuptime') + def showUptime(): + return Response(str(self.getUptime())) + + self.httpServer = WSGIServer((self.host, bindPort), app, log=None, handler_class=FDSafeHandler) self.httpServer.serve_forever() def setPublicAPIInstance(self, inst): @@ -327,3 +460,29 @@ class API: return True except TypeError: return False + + def getUptime(self): + while True: + try: + return self._utils.getEpoch - startTime + except AttributeError: + # Don't error on race condition with startup + pass + + def getBlockData(self, bHash, decrypt=False, raw=False): + bl = Block(bHash, core=self._core) + if decrypt: + bl.decrypt() + if bl.isEncrypted and not bl.decrypted: + raise ValueError + + if not raw: + retData = {'meta':bl.bheader, 'metadata': bl.bmetadata, 'content': bl.bcontent} + for x in list(retData.keys()): + try: + retData[x] = retData[x].decode() + except AttributeError: + pass + return json.dumps(retData) + else: + return bl.raw diff --git a/onionr/communicator2.py b/onionr/communicator.py similarity index 84% rename from onionr/communicator2.py rename to onionr/communicator.py index 5d045a69..1b8dc7fd 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator.py @@ -21,15 +21,17 @@ ''' import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block -import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs +import onionrdaemontools, onionrsockets, onionr, onionrproofs, proofofmemory import binascii from dependencies import secrets from defusedxml import minidom - +config.reload() class OnionrCommunicatorDaemon: - def __init__(self, debug, developmentMode): + def __init__(self, onionrInst, proxyPort, developmentMode=config.get('general.dev_mode', False)): + onionrInst.communicatorInst = self # configure logger and stuff onionr.Onionr.setupConfig('data/', self = self) + self.proxyPort = proxyPort self.isOnline = True # Assume we're connected to the internet @@ -37,8 +39,8 @@ class OnionrCommunicatorDaemon: self.timers = [] # initalize core with Tor socks port being 3rd argument - self.proxyPort = sys.argv[2] - self._core = core.Core(torPort=self.proxyPort) + self.proxyPort = proxyPort + self._core = onionrInst.onionrCore # intalize NIST beacon salt and time self.nistSaltTimestamp = 0 @@ -49,9 +51,6 @@ class OnionrCommunicatorDaemon: # loop time.sleep delay in seconds self.delay = 1 - # time app started running for info/statistics purposes - self.startTime = self._core._utils.getEpoch() - # lists of connected peers and peers we know we can't reach currently self.onlinePeers = [] self.offlinePeers = [] @@ -66,7 +65,7 @@ class OnionrCommunicatorDaemon: self.shutdown = False # list of new blocks to download, added to when new block lists are fetched from peers - self.blockQueue = [] + self.blockQueue = {} # list of blocks currently downloading, avoid s self.currentDownloading = [] @@ -74,6 +73,9 @@ class OnionrCommunicatorDaemon: # timestamp when the last online node was seen self.lastNodeSeen = None + # Dict of time stamps for peer's block list lookup times, to avoid downloading full lists all the time + self.dbTimestamps = {} + # Clear the daemon queue for any dead messages if os.path.exists(self._core.queueDB): self._core.clearDaemonQueue() @@ -81,33 +83,36 @@ class OnionrCommunicatorDaemon: # Loads in and starts the enabled plugins plugins.reload() + self.proofofmemory = proofofmemory.ProofOfMemory(self) + # daemon tools are misc daemon functions, e.g. announce to online peers # intended only for use by OnionrCommunicatorDaemon self.daemonTools = onionrdaemontools.DaemonTools(self) - self._chat = onionrchat.OnionrChat(self) + # time app started running for info/statistics purposes + self.startTime = self._core._utils.getEpoch() - if debug or developmentMode: + if developmentMode: OnionrCommunicatorTimers(self, self.heartbeat, 30) # Set timers, function reference, seconds # requiresPeer True means the timer function won't fire if we have no connected peers peerPoolTimer = OnionrCommunicatorTimers(self, self.getOnlinePeers, 60, maxThreads=1) - OnionrCommunicatorTimers(self, self.runCheck, 1) + OnionrCommunicatorTimers(self, self.runCheck, 2, maxThreads=1) OnionrCommunicatorTimers(self, self.lookupBlocks, self._core.config.get('timers.lookupBlocks'), requiresPeer=True, maxThreads=1) - OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True) + OnionrCommunicatorTimers(self, self.getBlocks, self._core.config.get('timers.getBlocks'), requiresPeer=True, maxThreads=2) OnionrCommunicatorTimers(self, self.clearOfflinePeer, 58) OnionrCommunicatorTimers(self, self.daemonTools.cleanOldBlocks, 65) OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True) OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1) OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1) - OnionrCommunicatorTimers(self, self.detectAPICrash, 5, maxThreads=1) + OnionrCommunicatorTimers(self, self.detectAPICrash, 30, maxThreads=1) deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) if config.get('general.security_level') == 0: - announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 86400, requiresPeer=True, maxThreads=1) + announceTimer = OnionrCommunicatorTimers(self, self.daemonTools.announceNode, 3600, requiresPeer=True, maxThreads=1) announceTimer.count = (announceTimer.frequency - 120) else: logger.debug('Will not announce node.') @@ -125,8 +130,6 @@ class OnionrCommunicatorDaemon: self.socketServer.start() self.socketClient = onionrsockets.OnionrSocketClient(self._core) - # Loads chat messages into memory - threading.Thread(target=self._chat.chatHandler).start() # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking try: @@ -136,6 +139,9 @@ class OnionrCommunicatorDaemon: break i.processTimer() time.sleep(self.delay) + # Debug to print out used FDs (regular and net) + #proc = psutil.Process() + #print(proc.open_files(), len(psutil.net_connections())) except KeyboardInterrupt: self.shutdown = True pass @@ -164,7 +170,9 @@ class OnionrCommunicatorDaemon: existingBlocks = self._core.getBlockList() triedPeers = [] # list of peers we've tried this time around maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion + lastLookupTime = 0 # Last time we looked up a particular peer's list for i in range(tryAmount): + listLookupCommand = 'getblocklist' # This is defined here to reset it each time if len(self.blockQueue) >= maxBacklog: break if not self.isOnline: @@ -186,11 +194,21 @@ class OnionrCommunicatorDaemon: triedPeers.append(peer) if newDBHash != self._core.getAddressInfo(peer, 'DBHash'): self._core.setAddressInfo(peer, 'DBHash', newDBHash) + # Get the last time we looked up a peer's stamp to only fetch blocks since then. + # Saved in memory only for privacy reasons try: - newBlocks = self.peerAction(peer, 'getblocklist') # get list of new block hashes + lastLookupTime = self.dbTimestamps[peer] + except KeyError: + lastLookupTime = 0 + else: + listLookupCommand += '?date=%s' % (lastLookupTime,) + try: + newBlocks = self.peerAction(peer, listLookupCommand) # get list of new block hashes except Exception as error: logger.warn('Could not get new blocks from %s.' % peer, error = error) newBlocks = False + else: + self.dbTimestamps[peer] = self._core._utils.getRoundedEpoch(roundS=60) if newBlocks != False: # if request was a success for i in newBlocks.split('\n'): @@ -198,15 +216,24 @@ class OnionrCommunicatorDaemon: # if newline seperated string is valid hash if not i in existingBlocks: # if block does not exist on disk and is not already in block queue - if i not in self.blockQueue and not self._core._blacklist.inBlacklist(i): - if onionrproofs.hashMeetsDifficulty(i): - self.blockQueue.append(i) # add blocks to download queue + if i not in self.blockQueue: + if onionrproofs.hashMeetsDifficulty(i) and not self._core._blacklist.inBlacklist(i): + if len(self.blockQueue) <= 1000000: + self.blockQueue[i] = [peer] # add blocks to download queue + else: + if peer not in self.blockQueue[i]: + self.blockQueue[i].append(peer) self.decrementThreadCount('lookupBlocks') return def getBlocks(self): '''download new blocks in queue''' - for blockHash in self.blockQueue: + for blockHash in list(self.blockQueue): + triedQueuePeers = [] # List of peers we've tried for a block + try: + blockPeers = list(self.blockQueue[blockHash]) + except KeyError: + blockPeers = [] removeFromQueue = True if self.shutdown or not self.isOnline: # Exit loop if shutting down or offline @@ -217,15 +244,24 @@ class OnionrCommunicatorDaemon: continue if blockHash in self._core.getBlockList(): logger.debug('Block %s is already saved.' % (blockHash,)) - self.blockQueue.remove(blockHash) + try: + del self.blockQueue[blockHash] + except KeyError: + pass continue if self._core._blacklist.inBlacklist(blockHash): continue if self._core._utils.storageCounter.isFull(): break self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block - logger.info("Attempting to download %s..." % blockHash) - peerUsed = self.pickOnlinePeer() + if len(blockPeers) == 0: + peerUsed = self.pickOnlinePeer() + else: + blockPeers = self._core._crypto.randomShuffle(blockPeers) + peerUsed = blockPeers.pop(0) + + if not self.shutdown and peerUsed.strip() != '': + logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed)) content = self.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata) if content != False and len(content) > 0: try: @@ -247,7 +283,7 @@ class OnionrCommunicatorDaemon: metadata = metas[0] if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce if self._core._crypto.verifyPow(content): # check if POW is enough/correct - logger.info('Attempting to save block %s...' % blockHash) + logger.info('Attempting to save block %s...' % blockHash[:12]) try: self._core.setData(content) except onionrexceptions.DiskAllocationReached: @@ -273,11 +309,15 @@ class OnionrCommunicatorDaemon: pass # Punish peer for sharing invalid block (not always malicious, but is bad regardless) onionrpeers.PeerProfiles(peerUsed, self._core).addScore(-50) - logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash) + if tempHash != 'ed55e34cb828232d6c14da0479709bfa10a0923dca2b380496e6b2ed4f7a0253': + # Dumb hack for 404 response from peer. Don't log it if 404 since its likely not malicious or a critical error. + logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash) + else: + removeFromQueue = False # Don't remove from queue if 404 if removeFromQueue: try: - self.blockQueue.remove(blockHash) # remove from block queue both if success or false - except ValueError: + del self.blockQueue[blockHash] # remove from block queue both if success or false + except KeyError: pass self.currentDownloading.remove(blockHash) self.decrementThreadCount('getBlocks') @@ -401,6 +441,10 @@ class OnionrCommunicatorDaemon: del self.connectTimes[peer] except KeyError: pass + try: + del self.dbTimestamps[peer] + except KeyError: + pass try: self.onlinePeers.remove(peer) except ValueError: @@ -459,10 +503,12 @@ class OnionrCommunicatorDaemon: retData = onionrpeers.PeerProfiles(peer, self._core) return retData + def getUptime(self): + return self._core._utils.getEpoch() - self.startTime + def heartbeat(self): '''Show a heartbeat debug message''' - currentTime = self._core._utils.getEpoch() - self.startTime - logger.debug('Heartbeat. Node running for %s.' % self.daemonTools.humanReadableTime(currentTime)) + logger.debug('Heartbeat. Node running for %s.' % self.daemonTools.humanReadableTime(self.getUptime())) self.decrementThreadCount('heartbeat') def daemonCommands(self): @@ -470,7 +516,7 @@ class OnionrCommunicatorDaemon: Process daemon commands from daemonQueue ''' cmd = self._core.daemonQueue() - + response = '' if cmd is not False: events.event('daemon_command', onionr = None, data = {'cmd' : cmd}) if cmd[0] == 'shutdown': @@ -484,7 +530,11 @@ class OnionrCommunicatorDaemon: logger.debug('Status check; looks good.') open(self._core.dataDir + '.runcheck', 'w+').close() elif cmd[0] == 'connectedPeers': - self.printOnlinePeers() + response = '\n'.join(list(self.onlinePeers)).strip() + if response == '': + response = 'none' + elif cmd[0] == 'localCommand': + response = self._core._utils.localCommand(cmd[1]) elif cmd[0] == 'pex': for i in self.timers: if i.timerFunction.__name__ == 'lookupAdders': @@ -505,6 +555,11 @@ class OnionrCommunicatorDaemon: else: logger.info('Recieved daemonQueue command:' + cmd[0]) + if cmd[0] not in ('', None): + if response != '': + self._core._utils.localCommand('queueResponseAdd/' + cmd[4], post=True, postData={'data': response}) + response = '' + self.decrementThreadCount('daemonCommands') def uploadBlock(self): @@ -512,13 +567,14 @@ class OnionrCommunicatorDaemon: # when inserting a block, we try to upload it to a few peers to add some deniability triedPeers = [] finishedUploads = [] + self.blocksToUpload = self._core._crypto.randomShuffle(self.blocksToUpload) if len(self.blocksToUpload) != 0: for bl in self.blocksToUpload: if not self._core._utils.validateHash(bl): logger.warn('Requested to upload invalid block') self.decrementThreadCount('uploadBlock') return - for i in range(max(len(self.onlinePeers), 2)): + for i in range(min(len(self.onlinePeers), 6)): peer = self.pickOnlinePeer() if peer in triedPeers: continue @@ -534,7 +590,6 @@ class OnionrCommunicatorDaemon: if not self._core._utils.doPostRequest(url, data=data, proxyType=proxyType) == False: self._core._utils.localCommand('waitforshare/' + bl) finishedUploads.append(bl) - break for x in finishedUploads: try: self.blocksToUpload.remove(x) @@ -610,18 +665,5 @@ class OnionrCommunicatorTimers: self.count = -1 # negative 1 because its incremented at bottom self.count += 1 -shouldRun = False -debug = True -developmentMode = False -if config.get('general.dev_mode', True): - developmentMode = True -try: - if sys.argv[1] == 'run': - shouldRun = True -except IndexError: - pass -if shouldRun: - try: - OnionrCommunicatorDaemon(debug, developmentMode) - except Exception as e: - logger.error('Error occured in Communicator', error = e, timestamp = False) +def startCommunicator(onionrInst, proxyPort): + OnionrCommunicatorDaemon(onionrInst, proxyPort) \ No newline at end of file diff --git a/onionr/config.py b/onionr/config.py index 7c986b83..960286f6 100644 --- a/onionr/config.py +++ b/onionr/config.py @@ -105,7 +105,8 @@ def check(): open(get_config_file(), 'a', encoding="utf8").close() save() except: - logger.warn('Failed to check configuration file.') + pass + #logger.debug('Failed to check configuration file.') def save(): ''' @@ -129,7 +130,8 @@ def reload(): with open(get_config_file(), 'r', encoding="utf8") as configfile: set_config(json.loads(configfile.read())) except: - logger.warn('Failed to parse configuration file.') + pass + #logger.debug('Failed to parse configuration file.') def get_config(): ''' diff --git a/onionr/core.py b/onionr/core.py index 1e698a4c..8634ab1f 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -17,12 +17,13 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import sqlite3, os, sys, time, math, base64, tarfile, nacl, logger, json, netcontroller, math, config +import sqlite3, os, sys, time, math, base64, tarfile, nacl, logger, json, netcontroller, math, config, uuid from onionrblockapi import Block -import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues -import onionrblacklist, onionrchat, onionrusers -import dbcreator +import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions +import onionrblacklist, onionrusers +import dbcreator, onionrstorage, serializeddata +from etc import onionrvalues if sys.version_info < (3, 6): try: @@ -45,10 +46,12 @@ class Core: self.dataDir = 'data/' try: + self.onionrInst = None self.queueDB = self.dataDir + 'queue.db' self.peerDB = self.dataDir + 'peers.db' self.blockDB = self.dataDir + 'blocks.db' self.blockDataLocation = self.dataDir + 'blocks/' + self.blockDataDB = self.blockDataLocation + 'block-data.db' self.publicApiHostFile = self.dataDir + 'public-host.txt' self.privateApiHostFile = self.dataDir + 'private-host.txt' self.addressDB = self.dataDir + 'address.db' @@ -97,9 +100,11 @@ class Core: logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation) self._utils = onionrutils.OnionrUtils(self) + self.blockCache = onionrstorage.BlockCache() # Initialize the crypto object self._crypto = onionrcrypto.OnionrCrypto(self) self._blacklist = onionrblacklist.OnionrBlackList(self) + self.serializer = serializeddata.SerializedData(self) except Exception as error: logger.error('Failed to initialize core Onionr library.', error=error) @@ -127,7 +132,7 @@ class Core: events.event('pubkey_add', data = {'key': peerID}, onionr = None) - conn = sqlite3.connect(self.peerDB, timeout=10) + conn = sqlite3.connect(self.peerDB, timeout=30) hashID = self._crypto.pubKeyHashID(peerID) c = conn.cursor() t = (peerID, name, 'unknown', hashID, 0) @@ -157,7 +162,7 @@ class Core: if type(address) is None or len(address) == 0: return False if self._utils.validateID(address): - conn = sqlite3.connect(self.addressDB, timeout=10) + conn = sqlite3.connect(self.addressDB, timeout=30) c = conn.cursor() # check if address is in database # this is safe to do because the address is validated above, but we strip some chars here too just in case @@ -181,7 +186,7 @@ class Core: return True else: - logger.debug('Invalid ID: %s' % address) + #logger.debug('Invalid ID: %s' % address) return False def removeAddress(self, address): @@ -190,7 +195,7 @@ class Core: ''' if self._utils.validateID(address): - conn = sqlite3.connect(self.addressDB, timeout=10) + conn = sqlite3.connect(self.addressDB, timeout=30) c = conn.cursor() t = (address,) c.execute('Delete from adders where address=?;', t) @@ -210,7 +215,7 @@ class Core: ''' if self._utils.validateHash(block): - conn = sqlite3.connect(self.blockDB, timeout=10) + conn = sqlite3.connect(self.blockDB, timeout=30) c = conn.cursor() t = (block,) c.execute('Delete from hashes where hash=?;', t) @@ -258,7 +263,7 @@ class Core: raise Exception('Block db does not exist') if self._utils.hasBlock(newHash): return - conn = sqlite3.connect(self.blockDB, timeout=10) + conn = sqlite3.connect(self.blockDB, timeout=30) c = conn.cursor() currentTime = self._utils.getEpoch() + self._crypto.secrets.randbelow(301) if selfInsert or dataSaved: @@ -277,6 +282,7 @@ class Core: Simply return the data associated to a hash ''' + ''' try: # logger.debug('Opening %s' % (str(self.blockDataLocation) + str(hash) + '.dat')) dataFile = open(self.blockDataLocation + hash + '.dat', 'rb') @@ -284,6 +290,8 @@ class Core: dataFile.close() except FileNotFoundError: data = False + ''' + data = onionrstorage.getData(self, hash) return data @@ -308,10 +316,11 @@ class Core: #raise Exception("Data is already set for " + dataHash) else: if self._utils.storageCounter.addBytes(dataSize) != False: - blockFile = open(blockFileName, 'wb') - blockFile.write(data) - blockFile.close() - conn = sqlite3.connect(self.blockDB, timeout=10) + #blockFile = open(blockFileName, 'wb') + #blockFile.write(data) + #blockFile.close() + onionrstorage.store(self, data, blockHash=dataHash) + conn = sqlite3.connect(self.blockDB, timeout=30) c = conn.cursor() c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = ?;", (dataHash,)) conn.commit() @@ -334,10 +343,10 @@ class Core: if not os.path.exists(self.queueDB): self.dbCreate.createDaemonDB() else: - conn = sqlite3.connect(self.queueDB, timeout=10) + conn = sqlite3.connect(self.queueDB, timeout=30) c = conn.cursor() try: - for row in c.execute('SELECT command, data, date, min(ID) FROM commands group by id'): + for row in c.execute('SELECT command, data, date, min(ID), responseID FROM commands group by id'): retData = row break except sqlite3.OperationalError: @@ -352,34 +361,58 @@ class Core: return retData - def daemonQueueAdd(self, command, data=''): + def daemonQueueAdd(self, command, data='', responseID=''): ''' Add a command to the daemon queue, used by the communication daemon (communicator.py) ''' retData = True - # Intended to be used by the web server date = self._utils.getEpoch() - conn = sqlite3.connect(self.queueDB, timeout=10) + conn = sqlite3.connect(self.queueDB, timeout=30) c = conn.cursor() - t = (command, data, date) + t = (command, data, date, responseID) try: - c.execute('INSERT INTO commands (command, data, date) VALUES(?, ?, ?)', t) + c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t) conn.commit() - conn.close() except sqlite3.OperationalError: retData = False self.daemonQueue() events.event('queue_push', data = {'command': command, 'data': data}, onionr = None) + conn.close() return retData + def daemonQueueGetResponse(self, responseID=''): + ''' + Get a response sent by communicator to the API, by requesting to the API + ''' + assert len(responseID) > 0 + resp = self._utils.localCommand('queueResponse/' + responseID) + return resp + + def daemonQueueWaitForResponse(self, responseID='', checkFreqSecs=1): + resp = 'failure' + while resp == 'failure': + resp = self.daemonQueueGetResponse(responseID) + time.sleep(1) + return resp + + def daemonQueueSimple(self, command, data='', checkFreqSecs=1): + ''' + A simplified way to use the daemon queue. Will register a command (with optional data) and wait, return the data + Not always useful, but saves time + LOC in some cases. + This is a blocking function, so be careful. + ''' + responseID = str(uuid.uuid4()) # generate unique response ID + self.daemonQueueAdd(command, data=data, responseID=responseID) + return self.daemonQueueWaitForResponse(responseID, checkFreqSecs) + def clearDaemonQueue(self): ''' Clear the daemon queue (somewhat dangerous) ''' - conn = sqlite3.connect(self.queueDB, timeout=10) + conn = sqlite3.connect(self.queueDB, timeout=30) c = conn.cursor() try: @@ -393,11 +426,11 @@ class Core: return - def listAdders(self, randomOrder=True, i2p=True): + def listAdders(self, randomOrder=True, i2p=True, recent=0): ''' Return a list of addresses ''' - conn = sqlite3.connect(self.addressDB, timeout=10) + conn = sqlite3.connect(self.addressDB, timeout=30) c = conn.cursor() if randomOrder: addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();') @@ -405,8 +438,17 @@ class Core: addresses = c.execute('SELECT * FROM adders;') addressList = [] for i in addresses: + if len(i[0].strip()) == 0: + continue addressList.append(i[0]) conn.close() + testList = list(addressList) # create new list to iterate + for address in testList: + try: + if recent > 0 and (self._utils.getEpoch() - self.getAddressInfo(address, 'lastConnect')) > recent: + raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0 + except TypeError: + addressList.remove(address) return addressList def listPeers(self, randomOrder=True, getPow=False, trust=0): @@ -416,7 +458,7 @@ class Core: randomOrder determines if the list should be in a random order trust sets the minimum trust to list ''' - conn = sqlite3.connect(self.peerDB, timeout=10) + conn = sqlite3.connect(self.peerDB, timeout=30) c = conn.cursor() payload = '' @@ -465,7 +507,7 @@ class Core: trust int 4 hashID text 5 ''' - conn = sqlite3.connect(self.peerDB, timeout=10) + conn = sqlite3.connect(self.peerDB, timeout=30) c = conn.cursor() command = (peer,) @@ -491,7 +533,7 @@ class Core: Update a peer for a key ''' - conn = sqlite3.connect(self.peerDB, timeout=10) + conn = sqlite3.connect(self.peerDB, timeout=30) c = conn.cursor() command = (data, peer) @@ -523,7 +565,7 @@ class Core: introduced 10 ''' - conn = sqlite3.connect(self.addressDB, timeout=10) + conn = sqlite3.connect(self.addressDB, timeout=30) c = conn.cursor() command = (address,) @@ -548,38 +590,41 @@ class Core: Update an address for a key ''' - conn = sqlite3.connect(self.addressDB, timeout=10) + conn = sqlite3.connect(self.addressDB, timeout=30) c = conn.cursor() command = (data, address) - + if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'DBHash', 'failure', 'powValue', 'lastConnect', 'lastConnectAttempt', 'trust', 'introduced'): raise Exception("Got invalid database key when setting address info") else: c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command) conn.commit() - conn.close() + conn.close() return - def getBlockList(self, unsaved = False): # TODO: Use unsaved?? + def getBlockList(self, dateRec = None, unsaved = False): ''' Get list of our blocks ''' + if dateRec == None: + dateRec = 0 - conn = sqlite3.connect(self.blockDB, timeout=10) + conn = sqlite3.connect(self.blockDB, timeout=30) c = conn.cursor() - if unsaved: - execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();' - else: - execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;' - + # if unsaved: + # execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();' + # else: + # execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;' + execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;' + args = (dateRec,) rows = list() - for row in c.execute(execute): + for row in c.execute(execute, args): for i in row: rows.append(i) - + conn.close() return rows def getBlockDate(self, blockHash): @@ -587,7 +632,7 @@ class Core: Returns the date a block was received ''' - conn = sqlite3.connect(self.blockDB, timeout=10) + conn = sqlite3.connect(self.blockDB, timeout=30) c = conn.cursor() execute = 'SELECT dateReceived FROM hashes WHERE hash=?;' @@ -595,7 +640,7 @@ class Core: for row in c.execute(execute, args): for i in row: return int(i) - + conn.close() return None def getBlocksByType(self, blockType, orderDate=True): @@ -603,7 +648,7 @@ class Core: Returns a list of blocks by the type ''' - conn = sqlite3.connect(self.blockDB, timeout=10) + conn = sqlite3.connect(self.blockDB, timeout=30) c = conn.cursor() if orderDate: @@ -617,12 +662,12 @@ class Core: for row in c.execute(execute, args): for i in row: rows.append(i) - + conn.close() return rows def getExpiredBlocks(self): '''Returns a list of expired blocks''' - conn = sqlite3.connect(self.blockDB, timeout=10) + conn = sqlite3.connect(self.blockDB, timeout=30) c = conn.cursor() date = int(self._utils.getEpoch()) @@ -632,6 +677,7 @@ class Core: for row in c.execute(execute): for i in row: rows.append(i) + conn.close() return rows def setBlockType(self, hash, blockType): @@ -639,7 +685,7 @@ class Core: Sets the type of block ''' - conn = sqlite3.connect(self.blockDB, timeout=10) + conn = sqlite3.connect(self.blockDB, timeout=30) c = conn.cursor() c.execute("UPDATE hashes SET dataType = ? WHERE hash = ?;", (blockType, hash)) conn.commit() @@ -666,7 +712,7 @@ class Core: if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound', 'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'): return False - conn = sqlite3.connect(self.blockDB, timeout=10) + conn = sqlite3.connect(self.blockDB, timeout=30) c = conn.cursor() args = (data, hash) c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args) @@ -675,12 +721,15 @@ class Core: return True - def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None): + def insertBlock(self, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False): ''' Inserts a block into the network encryptType must be specified to encrypt a block ''' - + allocationReachedMessage = 'Cannot insert block, disk allocation reached.' + if self._utils.storageCounter.isFull(): + logger.error(allocationReachedMessage) + return False retData = False # check nonce dataNonce = self._utils.bytesToStr(self._crypto.sha3Hash(data)) @@ -719,15 +768,17 @@ class Core: pass if encryptType == 'asym': - try: - forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data) - data = forwardEncrypted[0] - meta['forwardEnc'] = True - except onionrexceptions.InvalidPubkey: - onionrusers.OnionrUser(self, asymPeer).generateForwardKey() - onionrusers.OnionrUser(self, asymPeer).generateForwardKey() - fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys()[0] - meta['newFSKey'] = fsKey[0] + if not disableForward and asymPeer != self._crypto.pubKey: + try: + forwardEncrypted = onionrusers.OnionrUser(self, asymPeer).forwardEncrypt(data) + data = forwardEncrypted[0] + meta['forwardEnc'] = True + except onionrexceptions.InvalidPubkey: + pass + #onionrusers.OnionrUser(self, asymPeer).generateForwardKey() + fsKey = onionrusers.OnionrUser(self, asymPeer).generateForwardKey() + #fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys().reverse() + meta['newFSKey'] = fsKey jsonMeta = json.dumps(meta) if sign: signature = self._crypto.edSign(jsonMeta.encode() + data, key=self._crypto.privKey, encodeResult=True) @@ -774,13 +825,18 @@ class Core: proof = onionrproofs.POW(metadata, data) payload = proof.waitForResult() if payload != False: - retData = self.setData(payload) - # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult - self._utils.localCommand('waitforshare/' + retData) - self.addToBlockDB(retData, selfInsert=True, dataSaved=True) - #self.setBlockType(retData, meta['type']) - self._utils.processBlockMetadata(retData) - self.daemonQueueAdd('uploadBlock', retData) + try: + retData = self.setData(payload) + except onionrexceptions.DiskAllocationReached: + logger.error(allocationReachedMessage) + retData = False + else: + # Tell the api server through localCommand to wait for the daemon to upload this block to make stastical analysis more difficult + self._utils.localCommand('waitforshare/' + retData) + self.addToBlockDB(retData, selfInsert=True, dataSaved=True) + #self.setBlockType(retData, meta['type']) + self._utils.processBlockMetadata(retData) + self.daemonQueueAdd('uploadBlock', retData) if retData != False: events.event('insertBlock', onionr = None, threaded = False) @@ -813,5 +869,4 @@ class Core: else: logger.error('Onionr daemon is not running.') return False - return diff --git a/onionr/dbcreator.py b/onionr/dbcreator.py index f2f12f07..f728254a 100644 --- a/onionr/dbcreator.py +++ b/onionr/dbcreator.py @@ -92,7 +92,7 @@ class DBCreator: expire int - block expire date in epoch ''' if os.path.exists(self.core.blockDB): - raise Exception("Block database already exists") + raise FileExistsError("Block database already exists") conn = sqlite3.connect(self.core.blockDB) c = conn.cursor() c.execute('''CREATE TABLE hashes( @@ -111,13 +111,26 @@ class DBCreator: conn.commit() conn.close() return + + def createBlockDataDB(self): + if os.path.exists(self.core.blockDataDB): + raise FileExistsError("Block data database already exists") + conn = sqlite3.connect(self.core.blockDataDB) + c = conn.cursor() + c.execute('''CREATE TABLE blockData( + hash text not null, + data blob not null + ); + ''') + conn.commit() + conn.close() def createForwardKeyDB(self): ''' Create the forward secrecy key db (*for *OUR* keys*) ''' if os.path.exists(self.core.forwardKeysFile): - raise Exception("Block database already exists") + raise FileExistsError("Block database already exists") conn = sqlite3.connect(self.core.forwardKeysFile) c = conn.cursor() c.execute('''CREATE TABLE myForwardKeys( @@ -139,7 +152,6 @@ class DBCreator: conn = sqlite3.connect(self.core.queueDB, timeout=10) c = conn.cursor() # Create table - c.execute('''CREATE TABLE commands - (id integer primary key autoincrement, command text, data text, date text)''') + c.execute('''CREATE TABLE commands (id integer primary key autoincrement, command text, data text, date text, responseID text)''') conn.commit() conn.close() \ No newline at end of file diff --git a/onionr/onionrvalues.py b/onionr/etc/onionrvalues.py similarity index 100% rename from onionr/onionrvalues.py rename to onionr/etc/onionrvalues.py diff --git a/onionr/pgpwords.py b/onionr/etc/pgpwords.py similarity index 100% rename from onionr/pgpwords.py rename to onionr/etc/pgpwords.py diff --git a/onionr/logger.py b/onionr/logger.py index 7a8172b3..7cb409a1 100644 --- a/onionr/logger.py +++ b/onionr/logger.py @@ -132,8 +132,11 @@ def raw(data, fd = sys.stdout, sensitive = False): if get_settings() & OUTPUT_TO_CONSOLE: ts = fd.write('%s\n' % data) if get_settings() & OUTPUT_TO_FILE and not sensitive: - with open(_outputfile, "a+") as f: - f.write(colors.filter(data) + '\n') + try: + with open(_outputfile, "a+") as f: + f.write(colors.filter(data) + '\n') + except OSError: + pass def log(prefix, data, color = '', timestamp=True, fd = sys.stdout, prompt = True, sensitive = False): ''' diff --git a/onionr/netcontroller.py b/onionr/netcontroller.py index c87b5e46..c415c1a9 100644 --- a/onionr/netcontroller.py +++ b/onionr/netcontroller.py @@ -22,6 +22,7 @@ import subprocess, os, random, sys, logger, time, signal, config, base64, socket from stem.control import Controller from onionrblockapi import Block from dependencies import secrets +from shutil import which def getOpenPort(): # taken from (but modified) https://stackoverflow.com/a/2838309 @@ -31,6 +32,14 @@ def getOpenPort(): port = s.getsockname()[1] s.close() return port + +def torBinary(): + '''Return tor binary path or none if not exists''' + torPath = './tor' + if not os.path.exists(torPath): + torPath = which('tor') + return torPath + class NetController: ''' This class handles hidden service setup on Tor and I2P diff --git a/onionr/onionr.py b/onionr/onionr.py index aadceb56..110aa6f2 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -21,18 +21,19 @@ along with this program. If not, see . ''' import sys -if sys.version_info[0] == 2 or sys.version_info[1] < 5: - print('Error, Onionr requires Python 3.5+') +MIN_PY_VERSION = 6 +if sys.version_info[0] == 2 or sys.version_info[1] < MIN_PY_VERSION: + print('Error, Onionr requires Python 3.%s+' % (MIN_PY_VERSION,)) sys.exit(1) import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass, sqlite3 -import webbrowser +import webbrowser, uuid, signal from threading import Thread import api, core, config, logger, onionrplugins as plugins, onionrevents as events import onionrutils -import netcontroller +import netcontroller, onionrstorage from netcontroller import NetController from onionrblockapi import Block -import onionrproofs, onionrexceptions, onionrusers +import onionrproofs, onionrexceptions, onionrusers, communicator try: from urllib3.contrib.socks import SOCKSProxyManager @@ -51,6 +52,7 @@ class Onionr: In general, external programs and plugins should not use this class. ''' self.userRunDir = os.getcwd() # Directory user runs the program from + self.killed = False try: os.chdir(sys.path[0]) except FileNotFoundError: @@ -66,13 +68,21 @@ class Onionr: # Load global configuration data data_exists = Onionr.setupConfig(self.dataDir, self = self) + if netcontroller.torBinary() is None: + logger.error('Tor is not installed') + sys.exit(1) + + self.communicatorInst = None self.onionrCore = core.Core() + self.onionrCore.onionrInst = self #self.deleteRunFiles() self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore) self.clientAPIInst = '' # Client http api instance self.publicAPIInst = '' # Public http api instance + signal.signal(signal.SIGTERM, self.exitSigterm) + # Handle commands self.debug = False # Whole application debugging @@ -177,6 +187,12 @@ class Onionr: 'add-site': self.addWebpage, 'addsite': self.addWebpage, + 'openhome': self.openHome, + 'open-home': self.openHome, + + 'export-block': self.exportBlock, + 'exportblock': self.exportBlock, + 'get-file': self.getFile, 'getfile': self.getFile, @@ -192,7 +208,6 @@ class Onionr: 'ui' : self.openUI, 'gui' : self.openUI, - 'chat': self.startChat, 'getpassword': self.printWebPassword, 'get-password': self.printWebPassword, @@ -203,8 +218,6 @@ class Onionr: 'getpasswd': self.printWebPassword, 'get-passwd': self.printWebPassword, - 'chat': self.startChat, - 'friend': self.friendCmd, 'add-id': self.addID, 'change-id': self.changeID @@ -237,7 +250,8 @@ class Onionr: 'introduce': 'Introduce your node to the public Onionr network', 'friend': '[add|remove] [public key/id]', 'add-id': 'Generate a new ID (key pair)', - 'change-id': 'Change active ID' + 'change-id': 'Change active ID', + 'open-home': 'Open your node\'s home/info screen' } # initialize plugins @@ -252,11 +266,39 @@ class Onionr: self.execute(command) return + + def exitSigterm(self, signum, frame): + self.killed = True ''' THIS SECTION HANDLES THE COMMANDS ''' + def exportBlock(self): + exportDir = self.dataDir + 'block-export/' + try: + assert self.onionrUtils.validateHash(sys.argv[2]) + except (IndexError, AssertionError): + logger.error('No valid block hash specified.') + sys.exit(1) + else: + bHash = sys.argv[2] + try: + path = sys.argv[3] + except (IndexError): + if not os.path.exists(exportDir): + if os.path.exists(self.dataDir): + os.mkdir(exportDir) + else: + logger.error('Onionr not initialized') + sys.exit(1) + path = exportDir + data = onionrstorage.getData(self.onionrCore, bHash) + with open('%s/%s.dat' % (exportDir, bHash), 'wb') as exportFile: + exportFile.write(data) + + + def showDetails(self): details = { 'Node Address' : self.get_hostname(), @@ -268,6 +310,14 @@ class Onionr: for detail in details: logger.info('%s%s: \n%s%s\n' % (logger.colors.fg.lightgreen, detail, logger.colors.fg.green, details[detail]), sensitive = True) + def openHome(self): + try: + url = self.onionrUtils.getClientAPIServer() + except FileNotFoundError: + logger.error('Onionr seems to not be running (could not get api host)') + else: + webbrowser.open_new_tab('http://%s/#%s' % (url, config.get('client.webpassword'))) + def addID(self): try: sys.argv[2] @@ -276,7 +326,8 @@ class Onionr: newID = self.onionrCore._crypto.keyManager.addKey()[0] else: logger.warn('Deterministic keys require random and long passphrases.') - logger.warn('If a good password is not used, your key can be easily stolen.') + logger.warn('If a good passphrase is not used, your key can be easily stolen.') + logger.warn('You should use a series of hard to guess words, see this for reference: https://www.xkcd.com/936/') pass1 = getpass.getpass(prompt='Enter at least %s characters: ' % (self.onionrCore._crypto.deterministicRequirement,)) pass2 = getpass.getpass(prompt='Confirm entry: ') if self.onionrCore._crypto.safeCompare(pass1, pass2): @@ -310,14 +361,6 @@ class Onionr: else: logger.error('Invalid key %s' % (key,)) - def startChat(self): - try: - data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'}) - except IndexError: - logger.error('Must specify peer to chat with.') - else: - self.onionrCore.daemonQueueAdd('startSocket', data) - def getCommands(self): return self.cmds @@ -351,46 +394,9 @@ class Onionr: except IndexError: logger.error('Friend ID is required.') except onionrexceptions.KeyNotKnown: - logger.error('That peer is not in our database') - else: - if action == 'add': - friend.setTrust(1) - logger.info('Added %s as friend.' % (friend.publicKey,)) - else: - friend.setTrust(0) - logger.info('Removed %s as friend.' % (friend.publicKey,)) - else: - logger.info('Syntax: friend add/remove/list [address]') - - - def friendCmd(self): - '''List, add, or remove friend(s) - Changes their peer DB entry. - ''' - friend = '' - try: - # Get the friend command - action = sys.argv[2] - except IndexError: - logger.info('Syntax: friend add/remove/list [address]') - else: - action = action.lower() - if action == 'list': - # List out peers marked as our friend - for friend in self.onionrCore.listPeers(randomOrder=False, trust=1): - if friend == self.onionrCore._crypto.pubKey: # do not list our key - continue - friendProfile = onionrusers.OnionrUser(self.onionrCore, friend) - logger.info(friend + ' - ' + friendProfile.getName()) - elif action in ('add', 'remove'): - try: - friend = sys.argv[3] - if not self.onionrUtils.validatePubKey(friend): - raise onionrexceptions.InvalidPubkey('Public key is invalid') + self.onionrCore.addPeer(friend) friend = onionrusers.OnionrUser(self.onionrCore, friend) - except IndexError: - logger.error('Friend ID is required.') - else: + finally: if action == 'add': friend.setTrust(1) logger.info('Added %s as friend.' % (friend.publicKey,)) @@ -400,6 +406,15 @@ class Onionr: else: logger.info('Syntax: friend add/remove/list [address]') + def deleteRunFiles(self): + try: + os.remove(self.onionrCore.publicApiHostFile) + except FileNotFoundError: + pass + try: + os.remove(self.onionrCore.privateApiHostFile) + except FileNotFoundError: + pass def deleteRunFiles(self): try: @@ -432,7 +447,21 @@ class Onionr: return def listConn(self): - self.onionrCore.daemonQueueAdd('connectedPeers') + randID = str(uuid.uuid4()) + self.onionrCore.daemonQueueAdd('connectedPeers', responseID=randID) + while True: + try: + time.sleep(3) + peers = self.onionrCore.daemonQueueGetResponse(randID) + except KeyboardInterrupt: + break + if not type(peers) is None: + if peers not in ('', 'failure', None): + if peers != False: + print(peers) + else: + print('Daemon probably not running. Unable to list connected peers.') + break def listPeers(self): logger.info('Peer transport address list:') @@ -720,8 +749,6 @@ class Onionr: Starts the Onionr communication daemon ''' - communicatorDaemon = './communicator2.py' - # remove runcheck if it exists if os.path.isfile('data/.runcheck'): logger.debug('Runcheck file found on daemon start, deleting in advance.') @@ -760,8 +787,12 @@ class Onionr: logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey)) time.sleep(1) - # TODO: make runable on windows - communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)]) + self.onionrCore.torPort = net.socksPort + communicatorThread = Thread(target=communicator.startCommunicator, args=(self, str(net.socksPort))) + communicatorThread.start() + + while self.communicatorInst is None: + time.sleep(0.1) # print nice header thing :) if config.get('general.display_header', True): @@ -776,17 +807,23 @@ class Onionr: events.event('daemon_start', onionr = self) try: while True: - time.sleep(5) - + time.sleep(3) + # Debug to print out used FDs (regular and net) + #proc = psutil.Process() + #print('api-files:',proc.open_files(), len(psutil.net_connections())) # Break if communicator process ends, so we don't have left over processes - if communicatorProc.poll() is not None: + if self.communicatorInst.shutdown: break + if self.killed: + break # Break out if sigterm for clean exit except KeyboardInterrupt: + pass + finally: self.onionrCore.daemonQueueAdd('shutdown') self.onionrUtils.localCommand('shutdown') + net.killTor() time.sleep(3) self.deleteRunFiles() - net.killTor() return def killDaemon(self): @@ -815,7 +852,7 @@ class Onionr: try: # define stats messages here - totalBlocks = len(Block.getBlocks()) + totalBlocks = len(self.onionrCore.getBlockList()) signedBlocks = len(Block.getBlocks(signed = True)) messages = { # info about local client @@ -938,7 +975,8 @@ class Onionr: logger.error('Block hash is invalid') return - Block.mergeChain(bHash, fileName) + with open(fileName, 'wb') as myFile: + myFile.write(base64.b64decode(Block(bHash, core=self.onionrCore).bcontent)) return def addWebpage(self): @@ -961,12 +999,9 @@ class Onionr: return logger.info('Adding file... this might take a long time.') try: - if singleBlock: - with open(filename, 'rb') as singleFile: - blockhash = self.onionrCore.insertBlock(base64.b64encode(singleFile.read()), header=blockType) - else: - blockhash = Block.createChain(file = filename) - logger.info('File %s saved in block %s.' % (filename, blockhash)) + with open(filename, 'rb') as singleFile: + blockhash = self.onionrCore.insertBlock(base64.b64encode(singleFile.read()), header=blockType) + logger.info('File %s saved in block %s' % (filename, blockhash)) except: logger.error('Failed to save file in block.', timestamp = False) else: diff --git a/onionr/onionrblacklist.py b/onionr/onionrblacklist.py index 8736f78f..a87163f3 100644 --- a/onionr/onionrblacklist.py +++ b/onionr/onionrblacklist.py @@ -39,7 +39,7 @@ class OnionrBlackList: for i in self._dbExecute("SELECT * FROM blacklist WHERE hash = ?", (hashed,)): retData = True # this only executes if an entry is present by that hash break - + return retData def _dbExecute(self, toExec, params = ()): @@ -82,7 +82,7 @@ class OnionrBlackList: return def clearDB(self): - self._dbExecute('''DELETE FROM blacklist;);''') + self._dbExecute('''DELETE FROM blacklist;''') def getList(self): data = self._dbExecute('SELECT * FROM blacklist') diff --git a/onionr/onionrblockapi.py b/onionr/onionrblockapi.py index e6506faf..09dd77ea 100644 --- a/onionr/onionrblockapi.py +++ b/onionr/onionrblockapi.py @@ -1,7 +1,7 @@ ''' Onionr - P2P Anonymous Storage Network - This class contains the OnionrBlocks class which is a class for working with Onionr blocks + This file contains the OnionrBlocks class which is a class for working with Onionr blocks ''' ''' This program is free software: you can redistribute it and/or modify @@ -19,13 +19,13 @@ ''' import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions, onionrusers -import json, os, sys, datetime, base64 +import json, os, sys, datetime, base64, onionrstorage class Block: blockCacheOrder = list() # NEVER write your own code that writes to this! blockCache = dict() # should never be accessed directly, look at Block.getCache() - def __init__(self, hash = None, core = None, type = None, content = None, expire=None): + def __init__(self, hash = None, core = None, type = None, content = None, expire=None, decrypt=False): # take from arguments # sometimes people input a bytes object instead of str in `hash` if (not hash is None) and isinstance(hash, bytes): @@ -51,23 +51,13 @@ class Block: self.decrypted = False self.signer = None self.validSig = False + self.autoDecrypt = decrypt # handle arguments if self.getCore() is None: self.core = onionrcore.Core() - # update the blocks' contents if it exists - if not self.getHash() is None: - if not self.core._utils.validateHash(self.hash): - logger.debug('Block hash %s is invalid.' % self.getHash()) - raise onionrexceptions.InvalidHexHash('Block hash is invalid.') - elif not self.update(): - logger.debug('Failed to open block %s.' % self.getHash()) - else: - pass - #logger.debug('Did not update block.') - - # logic + self.update() def decrypt(self, anonymous = True, encodedData = True): ''' @@ -91,6 +81,7 @@ class Block: self.bmetadata = json.loads(bmeta) self.signature = core._crypto.pubKeyDecrypt(self.signature, anonymous=anonymous, encodedData=encodedData) self.signer = core._crypto.pubKeyDecrypt(self.signer, anonymous=anonymous, encodedData=encodedData) + self.bheader['signer'] = self.signer.decode() self.signedData = json.dumps(self.bmetadata) + self.bcontent.decode() try: assert self.bmetadata['forwardEnc'] is True @@ -140,13 +131,15 @@ class Block: Outputs: - (bool): indicates whether or not the operation was successful ''' - try: # import from string blockdata = data # import from file if blockdata is None: + blockdata = onionrstorage.getData(self.core, self.getHash()).decode() + ''' + filelocation = file readfile = True @@ -164,13 +157,14 @@ class Block: filelocation = self.core.dataDir + 'blocks/%s.dat' % self.getHash() if readfile: - with open(filelocation, 'rb') as f: - blockdata = f.read().decode() + blockdata = onionrstorage.getData(self.core, self.getHash()).decode() + #with open(filelocation, 'rb') as f: + #blockdata = f.read().decode() self.blockFile = filelocation + ''' else: self.blockFile = None - # parse block self.raw = str(blockdata) self.bheader = json.loads(self.getRaw()[:self.getRaw().index('\n')]) @@ -198,6 +192,9 @@ class Block: if len(self.getRaw()) <= config.get('allocations.blockCache', 500000): self.cache() + + if self.autoDecrypt: + self.decrypt() return True except Exception as e: @@ -240,13 +237,16 @@ class Block: try: if self.isValid() is True: + ''' if (not self.getBlockFile() is None) and (recreate is True): - with open(self.getBlockFile(), 'wb') as blockFile: - blockFile.write(self.getRaw().encode()) + onionrstorage.store(self.core, self.getRaw().encode()) + #with open(self.getBlockFile(), 'wb') as blockFile: + # blockFile.write(self.getRaw().encode()) else: - self.hash = self.getCore().insertBlock(self.getContent(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire()) - - self.update() + ''' + self.hash = self.getCore().insertBlock(self.getRaw(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire()) + if self.hash != False: + self.update() return self.getHash() else: @@ -585,7 +585,7 @@ class Block: return list() - def mergeChain(child, file = None, maximumFollows = 32, core = None): + def mergeChain(child, file = None, maximumFollows = 1000, core = None): ''' Follows a child Block to its root parent Block, merging content @@ -632,7 +632,7 @@ class Block: blocks.append(block.getHash()) - buffer = '' + buffer = b'' # combine block contents for hash in blocks: @@ -641,100 +641,18 @@ class Block: contents = base64.b64decode(contents.encode()) if file is None: - buffer += contents.decode() + try: + buffer += contents.encode() + except AttributeError: + buffer += contents else: file.write(contents) + if file is not None: + file.close() return (None if not file is None else buffer) - def createChain(data = None, chunksize = 99800, file = None, type = 'chunk', sign = True, encrypt = False, verbose = False): - ''' - Creates a chain of blocks to store larger amounts of data - - The chunksize is set to 99800 because it provides the least amount of PoW for the most amount of data. - - Inputs: - - data (*): if `file` is None, the data to be stored in blocks - - file (file/str): the filename or file object to read from (or None to read `data` instead) - - chunksize (int): the number of bytes per block chunk - - type (str): the type header for each of the blocks - - sign (bool): whether or not to sign each block - - encrypt (str): the public key to encrypt to, or False to disable encryption - - verbose (bool): whether or not to return a tuple containing more info - - Outputs: - - if `verbose`: - - (tuple): - - (str): the child block hash - - (list): all block hashes associated with storing the file - - if not `verbose`: - - (str): the child block hash - ''' - - blocks = list() - - # initial datatype checks - if data is None and file is None: - return blocks - elif not (file is None or (isinstance(file, str) and os.path.exists(file))): - return blocks - elif isinstance(file, str): - file = open(file, 'rb') - if not isinstance(data, str): - data = str(data) - - if not file is None: - filesize = os.stat(file.name).st_size - offset = filesize % chunksize - maxtimes = int(filesize / chunksize) - - for times in range(0, maxtimes + 1): - # read chunksize bytes from the file (end -> beginning) - if times < maxtimes: - file.seek(- ((times + 1) * chunksize), 2) - content = file.read(chunksize) - else: - file.seek(0, 0) - content = file.read(offset) - - # encode it- python is really bad at handling certain bytes that - # are often present in binaries. - content = base64.b64encode(content).decode() - - # if it is the end of the file, exit - if not content: - break - - # create block - block = Block() - block.setType(type) - block.setContent(content) - block.setParent((blocks[-1] if len(blocks) != 0 else None)) - hash = block.save(sign = sign) - - # remember the hash in cache - blocks.append(hash) - elif not data is None: - for content in reversed([data[n:n + chunksize] for n in range(0, len(data), chunksize)]): - # encode chunk with base64 - content = base64.b64encode(content.encode()).decode() - - # create block - block = Block() - block.setType(type) - block.setContent(content) - block.setParent((blocks[-1] if len(blocks) != 0 else None)) - hash = block.save(sign = sign) - - # remember the hash in cache - blocks.append(hash) - - # return different things depending on verbosity - if verbose: - return (blocks[-1], blocks) - return blocks[-1] - - def exists(hash): + def exists(bHash): ''' Checks if a block is saved to file or not @@ -748,15 +666,20 @@ class Block: ''' # no input data? scrap it. - if hash is None: + if bHash is None: return False - + ''' if type(hash) == Block: blockfile = hash.getBlockFile() else: blockfile = onionrcore.Core().dataDir + 'blocks/%s.dat' % hash + ''' + if isinstance(bHash, Block): + bHash = bHash.getHash() + + ret = isinstance(onionrstorage.getData(onionrcore.Core(), bHash), type(None)) - return os.path.exists(blockfile) and os.path.isfile(blockfile) + return not ret def getCache(hash = None): # give a list of the hashes of the cached blocks @@ -789,7 +712,7 @@ class Block: if block.getHash() in Block.getCache() and not override: return False - # dump old cached blocks if the size exeeds the maximum + # dump old cached blocks if the size exceeds the maximum if sys.getsizeof(Block.blockCacheOrder) >= config.get('allocations.block_cache_total', 50000000): # 50MB default cache size del Block.blockCache[blockCacheOrder.pop(0)] diff --git a/onionr/onionrchat.py b/onionr/onionrchat.py deleted file mode 100644 index 84483295..00000000 --- a/onionr/onionrchat.py +++ /dev/null @@ -1,50 +0,0 @@ -''' - Onionr - P2P Anonymous Storage Network - - Onionr Chat Messages -''' -''' - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' -import logger, time - -class OnionrChat: - def __init__(self, communicatorInst): - '''OnionrChat uses onionrsockets (handled by the communicator) to exchange direct chat messages''' - self.communicator = communicatorInst - self._core = self.communicator._core - self._utils = self._core._utils - - self.chats = {} # {'peer': {'date': date, message': message}} - self.chatSend = {} - - def chatHandler(self): - while not self.communicator.shutdown: - for peer in self._core.socketServerConnData: - try: - assert self._core.socketReasons[peer] == "chat" - except (AssertionError, KeyError) as e: - logger.warn('Peer is not for chat') - continue - else: - self.chats[peer] = {'date': self._core.socketServerConnData[peer]['date'], 'data': self._core.socketServerConnData[peer]['data']} - logger.info("CHAT MESSAGE RECIEVED: %s" % self.chats[peer]['data']) - for peer in self.communicator.socketClient.sockets: - try: - logger.info(self.communicator.socketClient.connPool[peer]['data']) - self.communicator.socketClient.sendData(peer, "lol") - except: - pass - - time.sleep(2) diff --git a/onionr/onionrcrypto.py b/onionr/onionrcrypto.py index 04c821cd..074c8aa1 100644 --- a/onionr/onionrcrypto.py +++ b/onionr/onionrcrypto.py @@ -210,12 +210,9 @@ class OnionrCrypto: ops = nacl.pwhash.argon2id.OPSLIMIT_SENSITIVE mem = nacl.pwhash.argon2id.MEMLIMIT_SENSITIVE - key = kdf(nacl.secret.SecretBox.KEY_SIZE, passphrase, salt, opslimit=ops, memlimit=mem) - key = nacl.public.PrivateKey(key, nacl.encoding.RawEncoder()) - publicKey = key.public_key - - return (publicKey.encode(encoder=nacl.encoding.Base32Encoder()), - key.encode(encoder=nacl.encoding.Base32Encoder())) + key = kdf(32, passphrase, salt, opslimit=ops, memlimit=mem) # Generate seed for ed25519 key + key = nacl.signing.SigningKey(key) + return (key.verify_key.encode(nacl.encoding.Base32Encoder).decode(), key.encode(nacl.encoding.Base32Encoder).decode()) def pubKeyHashID(self, pubkey=''): '''Accept a ed25519 public key, return a truncated result of X many sha3_256 hash rounds''' @@ -268,8 +265,9 @@ class OnionrCrypto: blockHash = blockHash.decode() # bytes on some versions for some reason except AttributeError: pass - - difficulty = math.floor(dataLen / 1000000) + + difficulty = onionrproofs.getDifficultyForNewBlock(blockContent, ourBlock=False) + if difficulty < int(config.get('general.minimum_block_pow')): difficulty = int(config.get('general.minimum_block_pow')) mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode() @@ -283,5 +281,20 @@ class OnionrCrypto: return retData - def safeCompare(self, one, two): + @staticmethod + def safeCompare(one, two): return hmac.compare_digest(one, two) + + @staticmethod + def randomShuffle(theList): + myList = list(theList) + shuffledList = [] + myListLength = len(myList) + 1 + while myListLength > 0: + removed = secrets.randbelow(myListLength) + try: + shuffledList.append(myList.pop(removed)) + except IndexError: + pass + myListLength = len(myList) + return shuffledList \ No newline at end of file diff --git a/onionr/onionrdaemontools.py b/onionr/onionrdaemontools.py index 9f1ab2a2..faea0070 100644 --- a/onionr/onionrdaemontools.py +++ b/onionr/onionrdaemontools.py @@ -34,47 +34,47 @@ class DaemonTools: '''Announce our node to our peers''' retData = False announceFail = False - - # Announce to random online peers - for i in self.daemon.onlinePeers: - if not i in self.announceCache: - peer = i - break - else: - peer = self.daemon.pickOnlinePeer() - - ourID = self.daemon._core.hsAddress.strip() - - url = 'http://' + peer + '/announce' - data = {'node': ourID} - - combinedNodes = ourID + peer - existingRand = self.daemon._core.getAddressInfo(peer, 'powValue') - if type(existingRand) is type(None): - existingRand = '' - - if peer in self.announceCache: - data['random'] = self.announceCache[peer] - elif len(existingRand) > 0: - data['random'] = existingRand - else: - proof = onionrproofs.DataPOW(combinedNodes, forceDifficulty=4) - try: - data['random'] = base64.b64encode(proof.waitForResult()[1]) - except TypeError: - # Happens when we failed to produce a proof - logger.error("Failed to produce a pow for announcing to " + peer) - announceFail = True + if self.daemon._core.config.get('general.security_level', 0) == 0: + # Announce to random online peers + for i in self.daemon.onlinePeers: + if not i in self.announceCache: + peer = i + break else: - self.announceCache[peer] = data['random'] - if not announceFail: - logger.info('Announcing node to ' + url) - if self.daemon._core._utils.doPostRequest(url, data) == 'Success': - logger.info('Successfully introduced node to ' + peer) - retData = True - self.daemon._core.setAddressInfo(peer, 'introduced', 1) - self.daemon._core.setAddressInfo(peer, 'powValue', data['random']) - self.daemon.decrementThreadCount('announceNode') + peer = self.daemon.pickOnlinePeer() + + ourID = self.daemon._core.hsAddress.strip() + + url = 'http://' + peer + '/announce' + data = {'node': ourID} + + combinedNodes = ourID + peer + existingRand = self.daemon._core.getAddressInfo(peer, 'powValue') + if type(existingRand) is type(None): + existingRand = '' + + if peer in self.announceCache: + data['random'] = self.announceCache[peer] + elif len(existingRand) > 0: + data['random'] = existingRand + else: + proof = onionrproofs.DataPOW(combinedNodes, forceDifficulty=4) + try: + data['random'] = base64.b64encode(proof.waitForResult()[1]) + except TypeError: + # Happens when we failed to produce a proof + logger.error("Failed to produce a pow for announcing to " + peer) + announceFail = True + else: + self.announceCache[peer] = data['random'] + if not announceFail: + logger.info('Announcing node to ' + url) + if self.daemon._core._utils.doPostRequest(url, data) == 'Success': + logger.info('Successfully introduced node to ' + peer) + retData = True + self.daemon._core.setAddressInfo(peer, 'introduced', 1) + self.daemon._core.setAddressInfo(peer, 'powValue', data['random']) + self.daemon.decrementThreadCount('announceNode') return retData def netCheck(self): diff --git a/onionr/onionrexceptions.py b/onionr/onionrexceptions.py index a0c468a3..6763a172 100644 --- a/onionr/onionrexceptions.py +++ b/onionr/onionrexceptions.py @@ -53,6 +53,9 @@ class BlacklistedBlock(Exception): class DataExists(Exception): pass +class NoDataAvailable(Exception): + pass + class InvalidHexHash(Exception): '''When a string is not a valid hex string of appropriate length for a hash value''' pass diff --git a/onionr/onionrfragment.py b/onionr/onionrfragment.py new file mode 100644 index 00000000..c8386465 --- /dev/null +++ b/onionr/onionrfragment.py @@ -0,0 +1,73 @@ +''' + Onionr - P2P Anonymous Storage Network + + This file contains the OnionrFragment class which implements the fragment system +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' + +# onionr:10ch+10ch+10chgdecryptionkey +import core, sys, binascii, os + +FRAGMENT_SIZE = 0.25 +TRUNCATE_LENGTH = 30 + +class OnionrFragment: + def __init__(self, uri=None): + uri = uri.replace('onionr:', '') + count = 0 + blocks = [] + appendData = '' + key = '' + for x in uri: + if x == 'k': + key = uri[uri.index('k') + 1:] + appendData += x + if count == TRUNCATE_LENGTH: + blocks.append(appendData) + appendData = '' + count = 0 + count += 1 + self.key = key + self.blocks = blocks + return + + @staticmethod + def generateFragments(data=None, coreInst=None): + if coreInst is None: + coreInst = core.Core() + + key = os.urandom(32) + data = coreInst._crypto.symmetricEncrypt(data, key).decode() + blocks = [] + blockData = b"" + uri = "onionr:" + total = sys.getsizeof(data) + for x in data: + blockData += x.encode() + if round(len(blockData) / len(data), 3) > FRAGMENT_SIZE: + blocks.append(core.Core().insertBlock(blockData)) + blockData = b"" + + for bl in blocks: + uri += bl[:TRUNCATE_LENGTH] + uri += "k" + uri += binascii.hexlify(key).decode() + return (uri, key) + +if __name__ == '__main__': + uri = OnionrFragment.generateFragments("test")[0] + print(uri) + OnionrFragment(uri) \ No newline at end of file diff --git a/onionr/onionrgui.py b/onionr/onionrgui.py deleted file mode 100755 index 510ea594..00000000 --- a/onionr/onionrgui.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -from tkinter import * -import core -class OnionrGUI: - def __init__(self): - self.dataDir = "/programming/onionr/data/" - self.root = Tk() - self.root.geometry("450x250") - self.core = core.Core() - menubar = Menu(self.root) - - # create a pulldown menu, and add it to the menu bar - filemenu = Menu(menubar, tearoff=0) - filemenu.add_command(label="Open", command=None) - filemenu.add_command(label="Save", command=None) - filemenu.add_separator() - filemenu.add_command(label="Exit", command=self.root.quit) - menubar.add_cascade(label="File", menu=filemenu) - - settingsmenu = Menu(menubar, tearoff=0) - menubar.add_cascade(label="Settings", menu=settingsmenu) - - helpmenu = Menu(menubar, tearoff=0) - menubar.add_cascade(label="Help", menu=helpmenu) - - self.root.config(menu=menubar) - - self.menuFrame = Frame(self.root) - self.mainButton = Button(self.menuFrame, text="Main View") - self.mainButton.grid(row=0, column=0, padx=0, pady=2, sticky=N+W) - self.tabButton1 = Button(self.menuFrame, text="Mail") - self.tabButton1.grid(row=0, column=1, padx=0, pady=2, sticky=N+W) - self.tabButton2 = Button(self.menuFrame, text="Message Flow") - self.tabButton2.grid(row=0, column=3, padx=0, pady=2, sticky=N+W) - - self.menuFrame.grid(row=0, column=0, padx=2, pady=0, sticky=N+W) - - - self.idFrame = Frame(self.root) - - self.ourIDLabel = Label(self.idFrame, text="ID: ") - self.ourIDLabel.grid(row=2, column=0, padx=1, pady=1, sticky=N+W) - self.ourID = Entry(self.idFrame) - self.ourID.insert(0, self.core._crypto.pubKey) - self.ourID.grid(row=2, column=1, padx=1, pady=1, sticky=N+W) - self.ourID.config(state='readonly') - self.idFrame.grid(row=1, column=0, padx=2, pady=2, sticky=N+W) - - self.syncStatus = Label(self.root, text="Sync Status: 15/100") - self.syncStatus.place(relx=1.0, rely=1.0, anchor=S+E) - self.peerCount = Label(self.root, text="Connected Peers: 3") - self.peerCount.place(relx=0.0, rely=1.0, anchor='sw') - - self.root.wm_title("Onionr") - self.root.mainloop() - return - -OnionrGUI() \ No newline at end of file diff --git a/onionr/onionrpeers.py b/onionr/onionrpeers.py index e4793dfb..8a88d649 100644 --- a/onionr/onionrpeers.py +++ b/onionr/onionrpeers.py @@ -28,6 +28,7 @@ class PeerProfiles: self.friendSigCount = 0 self.success = 0 self.failure = 0 + self.connectTime = None if not isinstance(coreInst, core.Core): raise TypeError("coreInst must be a type of core.Core") @@ -35,6 +36,7 @@ class PeerProfiles: assert isinstance(self.coreInst, core.Core) self.loadScore() + self.getConnectTime() return def loadScore(self): @@ -44,7 +46,13 @@ class PeerProfiles: except (TypeError, ValueError) as e: self.success = 0 self.score = self.success - + + def getConnectTime(self): + try: + self.connectTime = int(self.coreInst.getAddressInfo(self.address, 'lastConnect')) + except (KeyError, ValueError, TypeError) as e: + pass + def saveScore(self): '''Save the node's score to the database''' self.coreInst.setAddressInfo(self.address, 'success', self.score) @@ -61,14 +69,20 @@ def getScoreSortedPeerList(coreInst): peerList = coreInst.listAdders() peerScores = {} + peerTimes = {} for address in peerList: # Load peer's profiles into a list profile = PeerProfiles(address, coreInst) peerScores[address] = profile.score + if not isinstance(profile.connectTime, type(None)): + peerTimes[address] = profile.connectTime + else: + peerTimes[address] = 9000 - # Sort peers by their score, greatest to least + # Sort peers by their score, greatest to least, and then last connected time peerList = sorted(peerScores, key=peerScores.get, reverse=True) + peerList = sorted(peerTimes, key=peerTimes.get, reverse=True) return peerList def peerCleanup(coreInst): diff --git a/onionr/onionrproofs.py b/onionr/onionrproofs.py index 9665a4cb..f1645a49 100644 --- a/onionr/onionrproofs.py +++ b/onionr/onionrproofs.py @@ -19,7 +19,57 @@ ''' import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json -import core, config +import core, onionrutils, config +import onionrblockapi + +def getDifficultyModifier(coreOrUtilsInst=None): + '''Accepts a core or utils instance returns + the difficulty modifier for block storage based + on a variety of factors, currently only disk use. + ''' + classInst = coreOrUtilsInst + retData = 0 + if isinstance(classInst, core.Core): + useFunc = classInst._utils.storageCounter.getPercent + elif isinstance(classInst, onionrutils.OnionrUtils): + useFunc = classInst.storageCounter.getPercent + else: + useFunc = core.Core()._utils.storageCounter.getPercent + + percentUse = useFunc() + + if percentUse >= 0.50: + retData += 1 + elif percentUse >= 0.75: + retData += 2 + elif percentUse >= 0.95: + retData += 3 + + return retData + +def getDifficultyForNewBlock(data, ourBlock=True): + ''' + Get difficulty for block. Accepts size in integer, Block instance, or str/bytes full block contents + ''' + retData = 0 + dataSize = 0 + if isinstance(data, onionrblockapi.Block): + dataSize = len(data.getRaw().encode('utf-8')) + elif isinstance(data, str): + dataSize = len(data.encode('utf-8')) + elif isinstance(data, bytes): + dataSize = len(data) + elif isinstance(data, int): + dataSize = data + else: + raise ValueError('not Block, str, or int') + if ourBlock: + minDifficulty = config.get('general.minimum_send_pow') + else: + minDifficulty = config.get('general.minimum_block_pow') + + retData = max(minDifficulty, math.floor(dataSize / 1000000)) + getDifficultyModifier() + return retData def getHashDifficulty(h): ''' @@ -55,6 +105,7 @@ class DataPOW: self.difficulty = 0 self.data = data self.threadCount = threadCount + self.rounds = 0 config.reload() if forceDifficulty == 0: @@ -96,6 +147,7 @@ class DataPOW: while self.hashing: rand = nacl.utils.random() token = nacl.hash.blake2b(rand + self.data).decode() + self.rounds += 1 #print(token) if self.puzzle == token[0:self.difficulty]: self.hashing = False @@ -106,6 +158,7 @@ class DataPOW: endTime = math.floor(time.time()) if self.reporting: logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True) + logger.debug('Round count: %s' % (self.rounds,)) self.result = (token, rand) def shutdown(self): @@ -146,18 +199,28 @@ class DataPOW: return result class POW: - def __init__(self, metadata, data, threadCount = 5): + def __init__(self, metadata, data, threadCount = 5, forceDifficulty=0, coreInst=None): self.foundHash = False self.difficulty = 0 self.data = data self.metadata = metadata self.threadCount = threadCount - dataLen = len(data) + len(json.dumps(metadata)) - self.difficulty = math.floor(dataLen / 1000000) - if self.difficulty <= 2: - self.difficulty = int(config.get('general.minimum_block_pow')) + try: + assert isinstance(coreInst, core.Core) + except AssertionError: + myCore = core.Core() + else: + myCore = coreInst + dataLen = len(data) + len(json.dumps(metadata)) + + if forceDifficulty > 0: + self.difficulty = forceDifficulty + else: + # Calculate difficulty. Dumb for now, may use good algorithm in the future. + self.difficulty = getDifficultyForNewBlock(dataLen) + try: self.data = self.data.encode() except AttributeError: @@ -167,8 +230,7 @@ class POW: self.mainHash = '0' * 64 self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))] - - myCore = core.Core() + for i in range(max(1, threadCount)): t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore)) t.start() diff --git a/onionr/onionrstorage.py b/onionr/onionrstorage.py new file mode 100644 index 00000000..8e2aae41 --- /dev/null +++ b/onionr/onionrstorage.py @@ -0,0 +1,90 @@ +''' + Onionr - P2P Anonymous Storage Network + + This file handles block storage, providing an abstraction for storing blocks between file system and database +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' +import core, sys, sqlite3, os, dbcreator + +DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option + +class BlockCache: + def __init__(self): + self.blocks = {} + def cleanCache(self): + while sys.getsizeof(self.blocks) > 100000000: + self.blocks.pop(list(self.blocks.keys())[0]) + +def dbCreate(coreInst): + try: + dbcreator.DBCreator(coreInst).createBlockDataDB() + except FileExistsError: + pass + +def _dbInsert(coreInst, blockHash, data): + assert isinstance(coreInst, core.Core) + dbCreate(coreInst) + conn = sqlite3.connect(coreInst.blockDataDB, timeout=10) + c = conn.cursor() + data = (blockHash, data) + c.execute('INSERT INTO blockData (hash, data) VALUES(?, ?);', data) + conn.commit() + conn.close() + +def _dbFetch(coreInst, blockHash): + assert isinstance(coreInst, core.Core) + dbCreate(coreInst) + conn = sqlite3.connect(coreInst.blockDataDB, timeout=10) + c = conn.cursor() + for i in c.execute('SELECT data from blockData where hash = ?', (blockHash,)): + return i[0] + conn.commit() + conn.close() + return None + +def store(coreInst, data, blockHash=''): + assert isinstance(coreInst, core.Core) + assert coreInst._utils.validateHash(blockHash) + ourHash = coreInst._crypto.sha3Hash(data) + if blockHash != '': + assert ourHash == blockHash + else: + blockHash = ourHash + + if DB_ENTRY_SIZE_LIMIT >= sys.getsizeof(data): + _dbInsert(coreInst, blockHash, data) + else: + with open('%s/%s.dat' % (coreInst.blockDataLocation, blockHash), 'wb') as blockFile: + blockFile.write(data) + coreInst.blockCache.cleanCache() + +def getData(coreInst, bHash): + assert isinstance(coreInst, core.Core) + assert coreInst._utils.validateHash(bHash) + + bHash = coreInst._utils.bytesToStr(bHash) + + # First check DB for data entry by hash + # if no entry, check disk + # If no entry in either, raise an exception + retData = None + fileLocation = '%s/%s.dat' % (coreInst.blockDataLocation, bHash) + if os.path.exists(fileLocation): + with open(fileLocation, 'rb') as block: + retData = block.read() + else: + retData = _dbFetch(coreInst, bHash) + return retData \ No newline at end of file diff --git a/onionr/onionrusers.py b/onionr/onionrusers.py index 5267112e..c5bf41d1 100644 --- a/onionr/onionrusers.py +++ b/onionr/onionrusers.py @@ -83,7 +83,7 @@ class OnionrUser: if self._core._utils.validatePubKey(forwardKey): retData = self._core._crypto.pubKeyEncrypt(data, forwardKey, encodedData=True, anonymous=True) else: - raise onionrexceptions.InvalidPubkey("No valid forward key available for this user") + raise onionrexceptions.InvalidPubkey("No valid forward secrecy key available for this user") #self.generateForwardKey() return (retData, forwardKey) @@ -169,7 +169,9 @@ class OnionrUser: def addForwardKey(self, newKey, expire=604800): if not self._core._utils.validatePubKey(newKey): - raise onionrexceptions.InvalidPubkey + raise onionrexceptions.InvalidPubkey(newKey) + if newKey in self._getForwardKeys(): + return False # Add a forward secrecy key for the peer conn = sqlite3.connect(self._core.peerDB, timeout=10) c = conn.cursor() diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index c6cfdfb9..b34eb948 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -24,7 +24,8 @@ from onionrblockapi import Block import onionrexceptions from onionr import API_VERSION import onionrevents -import pgpwords, onionrusers, storagecounter +import onionrusers, storagecounter +from etc import pgpwords if sys.version_info < (3, 6): try: import sha3 @@ -150,28 +151,43 @@ class OnionrUtils: except Exception as error: logger.error('Failed to read my address.', error = error) return None + + def getClientAPIServer(self): + retData = '' + try: + with open(self._core.privateApiHostFile, 'r') as host: + hostname = host.read() + except FileNotFoundError: + raise FileNotFoundError + else: + retData += '%s:%s' % (hostname, config.get('client.client.port')) + return retData - def localCommand(self, command, data='', silent = True): + def localCommand(self, command, data='', silent = True, post=False, postData = {}, maxWait=10): ''' Send a command to the local http API server, securely. Intended for local clients, DO NOT USE for remote peers. ''' - config.reload() self.getTimeBypassToken() # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless. hostname = '' + waited = 0 while hostname == '': try: - with open(self._core.privateApiHostFile, 'r') as host: - hostname = host.read() + hostname = self.getClientAPIServer() except FileNotFoundError: - print('wat') time.sleep(1) + waited += 1 + if waited == maxWait: + return False if data != '': data = '&data=' + urllib.parse.quote_plus(data) - payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data) + payload = 'http://%s/%s%s' % (hostname, command, data) try: - retData = requests.get(payload, headers={'token': config.get('client.webpassword')}).text + if post: + retData = requests.post(payload, data=postData, headers={'token': config.get('client.webpassword'), 'Connection':'close'}, timeout=(maxWait, 30)).text + else: + retData = requests.get(payload, headers={'token': config.get('client.webpassword'), 'Connection':'close'}, timeout=(maxWait, 30)).text except Exception as error: if not silent: logger.error('Failed to make local request (command: %s):%s' % (command, error)) @@ -365,6 +381,7 @@ class OnionrUtils: '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string''' # TODO, make this check sane sizes retData = False + maxClockDifference = 60 # convert to dict if it is json string if type(metadata) is str: @@ -393,13 +410,14 @@ class OnionrUtils: break if i == 'time': if not self.isIntegerString(metadata[i]): - logger.warn('Block metadata time stamp is not integer string') + logger.warn('Block metadata time stamp is not integer string or int') break - if (metadata[i] - self.getEpoch()) > 30: - logger.warn('Block metadata time stamp is set for the future, which is not allowed.') + isFuture = (metadata[i] - self.getEpoch()) + if isFuture > maxClockDifference: + logger.warn('Block timestamp is skewed to the future over the max %s: %s' (maxClockDifference, isFuture)) break if (self.getEpoch() - metadata[i]) > maxAge: - logger.warn('Block is older than allowed: %s' % (maxAge,)) + logger.warn('Block is outdated: %s' % (metadata[i],)) elif i == 'expire': try: assert int(metadata[i]) > self.getEpoch() @@ -443,7 +461,7 @@ class OnionrUtils: return retVal def isIntegerString(self, data): - '''Check if a string is a valid base10 integer''' + '''Check if a string is a valid base10 integer (also returns true if already an int)''' try: int(data) except ValueError: @@ -607,7 +625,7 @@ class OnionrUtils: proxies = {'http': 'http://127.0.0.1:4444'} else: return - headers = {'user-agent': 'PyOnionr'} + headers = {'user-agent': 'PyOnionr', 'Connection':'close'} try: proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)} r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30)) @@ -632,11 +650,11 @@ class OnionrUtils: proxies = {'http': 'http://127.0.0.1:4444'} else: return - headers = {'user-agent': 'PyOnionr'} + headers = {'user-agent': 'PyOnionr', 'Connection':'close'} response_headers = dict() try: proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)} - r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30)) + r = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30), ) # Check server is using same API version as us if not ignoreAPI: try: diff --git a/onionr/proofofmemory.py b/onionr/proofofmemory.py new file mode 100644 index 00000000..4b0b0fa7 --- /dev/null +++ b/onionr/proofofmemory.py @@ -0,0 +1,29 @@ +''' + Onionr - P2P Anonymous Storage Network + + This file handles proof of memory functionality +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' + +class ProofOfMemory: + def __init__(self, commInst): + self.communicator = commInst + return + + def checkRandomPeer(self): + return + def checkPeer(self, peer): + return \ No newline at end of file diff --git a/onionr/serializeddata.py b/onionr/serializeddata.py new file mode 100644 index 00000000..1587e74e --- /dev/null +++ b/onionr/serializeddata.py @@ -0,0 +1,43 @@ +''' + Onionr - P2P Anonymous Storage Network + + This module serializes various data pieces for use in other modules, in particular the web api +''' +''' + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +''' + +import core, api, uuid, json + +class SerializedData: + def __init__(self, coreInst): + ''' + Serialized data is in JSON format: + { + 'success': bool, + 'foo': 'bar', + etc + } + ''' + assert isinstance(coreInst, core.Core) + self._core = coreInst + + def getStats(self): + '''Return statistics about our node''' + stats = {} + stats['uptime'] = self._core.onionrInst.communicatorInst.getUptime() + stats['connectedNodes'] = '\n'.join(self._core.onionrInst.communicatorInst.onlinePeers) + stats['blockCount'] = len(self._core.getBlockList()) + stats['blockQueueCount'] = len(self._core.onionrInst.communicatorInst.blockQueue) + return json.dumps(stats) diff --git a/onionr/static-data/bootstrap-nodes.txt b/onionr/static-data/bootstrap-nodes.txt index e69de29b..93dec7ce 100644 --- a/onionr/static-data/bootstrap-nodes.txt +++ b/onionr/static-data/bootstrap-nodes.txt @@ -0,0 +1 @@ +dd3llxdp5q6ak3zmmicoy3jnodmroouv2xr7whkygiwp3rl7nf23gdad.onion \ No newline at end of file diff --git a/onionr/static-data/default-plugins/cliui/main.py b/onionr/static-data/default-plugins/cliui/main.py index 323b9419..59b59b0c 100644 --- a/onionr/static-data/default-plugins/cliui/main.py +++ b/onionr/static-data/default-plugins/cliui/main.py @@ -19,7 +19,7 @@ ''' # Imports some useful libraries -import logger, config, threading, time, uuid, subprocess +import logger, config, threading, time, uuid, subprocess, sys from onionrblockapi import Block plugin_name = 'cliui' @@ -31,11 +31,14 @@ class OnionrCLIUI: self.myCore = apiInst.get_core() return - def subCommand(self, command): + def subCommand(self, command, args=None): try: #subprocess.run(["./onionr.py", command]) #subprocess.Popen(['./onionr.py', command], stdin=subprocess.STD, stdout=subprocess.STDOUT, stderr=subprocess.STDOUT) - subprocess.call(['./onionr.py', command]) + if args != None: + subprocess.call(['./onionr.py', command, args]) + else: + subprocess.call(['./onionr.py', command]) except KeyboardInterrupt: pass @@ -48,12 +51,11 @@ class OnionrCLIUI: isOnline = 'No' firstRun = True choice = '' - - if self.myCore._utils.localCommand('ping') == 'pong': + if self.myCore._utils.localCommand('ping', maxWait=10) == 'pong!': firstRun = False while showMenu: - if self.myCore._utils.localCommand('ping') == 'pong': + if self.myCore._utils.localCommand('ping', maxWait=2) == 'pong!': isOnline = "Yes" else: isOnline = "No" @@ -62,8 +64,7 @@ class OnionrCLIUI: 1. Flow (Anonymous public chat, use at your own risk) 2. Mail (Secure email-like service) 3. File Sharing -4. User Settings -5. Quit (Does not shutdown daemon) +4. Quit (Does not shutdown daemon) ''') try: choice = input(">").strip().lower() @@ -75,13 +76,9 @@ class OnionrCLIUI: elif choice in ("2", "mail"): self.subCommand("mail") elif choice in ("3", "file sharing", "file"): - print("Not supported yet") - elif choice in ("4", "user settings", "settings"): - try: - self.setName() - except (KeyboardInterrupt, EOFError) as e: - pass - elif choice in ("5", "quit"): + filename = input("Enter full path to file: ").strip() + self.subCommand("addfile", filename) + elif choice in ("4", "quit"): showMenu = False elif choice == "": pass @@ -89,14 +86,6 @@ class OnionrCLIUI: logger.error("Invalid choice") return - def setName(self): - try: - name = input("Enter your name: ") - if name != "": - self.myCore.insertBlock("userInfo-" + str(uuid.uuid1()), sign=True, header='userInfo', meta={'name': name}) - except KeyboardInterrupt: - pass - return def on_init(api, data = None): ''' diff --git a/onionr/static-data/default-plugins/flow/main.py b/onionr/static-data/default-plugins/flow/main.py index 6f430e82..09826b79 100644 --- a/onionr/static-data/default-plugins/flow/main.py +++ b/onionr/static-data/default-plugins/flow/main.py @@ -54,9 +54,10 @@ class OnionrFlow: self.flowRunning = False expireTime = self.myCore._utils.getEpoch() + 43200 if len(message) > 0: - insertBL = Block(content = message, type = 'txt', expire=expireTime, core = self.myCore) - insertBL.setMetadata('ch', self.channel) - insertBL.save() + self.myCore.insertBlock(message, header='txt', expire=expireTime, meta={'ch': self.channel}) + #insertBL = Block(content = message, type = 'txt', expire=expireTime, core = self.myCore) + #insertBL.setMetadata('ch', self.channel) + #insertBL.save() logger.info("Flow is exiting, goodbye") return @@ -66,10 +67,13 @@ class OnionrFlow: time.sleep(1) try: while self.flowRunning: - for block in Block.getBlocks(type = 'txt', core = self.myCore): + for block in self.myCore.getBlocksByType('txt'): + block = Block(block) if block.getMetadata('ch') != self.channel: + #print('not chan', block.getMetadata('ch')) continue if block.getHash() in self.alreadyOutputed: + #print('already') continue if not self.flowRunning: break @@ -79,7 +83,7 @@ class OnionrFlow: content = self.myCore._utils.escapeAnsi(content.replace('\n', '\\n').replace('\r', '\\r').strip()) logger.info(block.getDate().strftime("%m/%d %H:%M") + ' - ' + logger.colors.reset + content, prompt = False) self.alreadyOutputed.append(block.getHash()) - time.sleep(5) + time.sleep(5) except KeyboardInterrupt: self.flowRunning = False diff --git a/onionr/static-data/default-plugins/metadataprocessor/main.py b/onionr/static-data/default-plugins/metadataprocessor/main.py index c0d3d38d..166249be 100644 --- a/onionr/static-data/default-plugins/metadataprocessor/main.py +++ b/onionr/static-data/default-plugins/metadataprocessor/main.py @@ -28,24 +28,6 @@ plugin_name = 'metadataprocessor' # event listeners -def _processUserInfo(api, newBlock): - ''' - Set the username for a particular user, from a signed block by them - ''' - myBlock = newBlock - peerName = myBlock.getMetadata('name') - try: - if len(peerName) > 20: - raise onionrexceptions.InvalidMetdata('Peer name specified is too large') - except TypeError: - pass - except onionrexceptions.InvalidMetadata: - pass - else: - if signer in self.api.get_core().listPeers(): - api.get_core().setPeerInfo(signer, 'name', peerName) - logger.info('%s is now using the name %s.' % (signer, api.get_utils().escapeAnsi(peerName))) - def _processForwardKey(api, myBlock): ''' Get the forward secrecy key specified by the user for us to use @@ -67,12 +49,8 @@ def on_processblocks(api): # Process specific block types - # userInfo blocks, such as for setting username - if blockType == 'userInfo': - if api.data['validSig'] == True: # we use == True for type safety - _processUserInfo(api, myBlock) # forwardKey blocks, add a new forward secrecy key for a peer - elif blockType == 'forwardKey': + if blockType == 'forwardKey': if api.data['validSig'] == True: _processForwardKey(api, myBlock) # socket blocks diff --git a/onionr/static-data/default-plugins/pms/main.py b/onionr/static-data/default-plugins/pms/main.py index 0cf7e2eb..35b85519 100644 --- a/onionr/static-data/default-plugins/pms/main.py +++ b/onionr/static-data/default-plugins/pms/main.py @@ -74,6 +74,7 @@ class OnionrMail: logger.info('Decrypting messages...') choice = '' displayList = [] + subject = '' # this could use a lot of memory if someone has recieved a lot of messages for blockHash in self.myCore.getBlocksByType('pm'): @@ -97,7 +98,12 @@ class OnionrMail: senderDisplay = senderKey blockDate = pmBlocks[blockHash].getDate().strftime("%m/%d %H:%M") - displayList.append('%s. %s - %s: %s' % (blockCount, blockDate, senderDisplay[:12], blockHash)) + try: + subject = pmBlocks[blockHash].bmetadata['subject'] + except KeyError: + subject = '' + + displayList.append('%s. %s - %s - <%s>: %s' % (blockCount, blockDate, senderDisplay[:12], subject[:10], blockHash)) while choice not in ('-q', 'q', 'quit'): for i in displayList: logger.info(i) @@ -188,6 +194,7 @@ class OnionrMail: def draftMessage(self, recip=''): message = '' newLine = '' + subject = '' entering = False if len(recip) == 0: entering = True @@ -207,22 +214,31 @@ class OnionrMail: else: # if -q or ctrl-c/d, exit function here, otherwise we successfully got the public key return - - logger.info('Enter your message, stop by entering -q on a new line.') + try: + subject = logger.readline('Message subject: ') + except (KeyboardInterrupt, EOFError): + pass + + cancelEnter = False + logger.info('Enter your message, stop by entering -q on a new line. -c to cancel') while newLine != '-q': try: newLine = input() except (KeyboardInterrupt, EOFError): - pass + cancelEnter = True + if newLine == '-c': + cancelEnter = True + break if newLine == '-q': continue newLine += '\n' message += newLine - logger.info('Inserting encrypted message as Onionr block....') + if not cancelEnter: + logger.info('Inserting encrypted message as Onionr block....') - blockID = self.myCore.insertBlock(message, header='pm', encryptType='asym', asymPeer=recip, sign=True) - self.sentboxTools.addToSent(blockID, recip, message) + blockID = self.myCore.insertBlock(message, header='pm', encryptType='asym', asymPeer=recip, sign=True, meta={'subject': subject}) + self.sentboxTools.addToSent(blockID, recip, message) def menu(self): choice = '' while True: diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index 13436852..b6cb3145 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -1,14 +1,14 @@ { "general" : { "dev_mode" : true, - "display_header" : true, - "minimum_block_pow": 5, - "minimum_send_pow": 5, + "display_header" : false, + "minimum_block_pow": 1, + "minimum_send_pow": 1, "socket_servers": false, "security_level": 0, "max_block_age": 2678400, - "public_key": "", - "use_new_api_server": false + "bypass_tor_check": false, + "public_key": "" }, "www" : { @@ -50,7 +50,7 @@ "file": { "output": true, - "path": "data/output.log" + "path": "output.log" }, "console" : { @@ -70,7 +70,7 @@ }, "allocations" : { - "disk" : 10000000000, + "disk" : 100000000, "net_total" : 1000000000, "blockCache" : 5000000, "blockCacheTotal" : 50000000 @@ -79,7 +79,7 @@ "peers" : { "minimum_score" : -100, "max_stored_peers" : 5000, - "max_connect" : 10 + "max_connect" : 1000 }, "timers" : { diff --git a/onionr/static-data/www/board/board.js b/onionr/static-data/www/board/board.js new file mode 100644 index 00000000..7f513357 --- /dev/null +++ b/onionr/static-data/www/board/board.js @@ -0,0 +1,58 @@ +webpassword = '' +requested = [] + +document.getElementById('webpassWindow').style.display = 'block'; + +var windowHeight = window.innerHeight; +document.getElementById('webpassWindow').style.height = windowHeight + "px"; + +function httpGet(theUrl) { + var xmlHttp = new XMLHttpRequest() + xmlHttp.open( "GET", theUrl, false ) // false for synchronous request + xmlHttp.setRequestHeader('token', webpassword) + xmlHttp.send( null ) + if (xmlHttp.status == 200){ + return xmlHttp.responseText + } + else{ + return ""; + } +} +function appendMessages(msg){ + el = document.createElement('div') + el.className = 'entry' + el.innerText = msg + document.getElementById('feed').appendChild(el) + document.getElementById('feed').appendChild(document.createElement('br')) +} + +function getBlocks(){ + if (document.getElementById('none') !== null){ + document.getElementById('none').remove(); + + } + var feedText = httpGet('/getblocksbytype/txt') + var blockList = feedText.split(',') + for (i = 0; i < blockList.length; i++){ + if (! requested.includes(blockList[i])){ + bl = httpGet('/gethtmlsafeblockdata/' + blockList[i]) + appendMessages(bl) + requested.push(blockList[i]) + } + } +} + +document.getElementById('registerPassword').onclick = function(){ + webpassword = document.getElementById('webpassword').value + if (httpGet('/ping') === 'pong!'){ + document.getElementById('webpassWindow').style.display = 'none' + getBlocks() + } + else{ + alert('Sorry, but that password appears invalid.') + } +} + +document.getElementById('refreshFeed').onclick = function(){ + getBlocks() +} \ No newline at end of file diff --git a/onionr/static-data/www/board/index.html b/onionr/static-data/www/board/index.html new file mode 100644 index 00000000..7486a910 --- /dev/null +++ b/onionr/static-data/www/board/index.html @@ -0,0 +1,22 @@ + + + + + + + OnionrBoard + + + + + + +
None Yet :)
+ + + \ No newline at end of file diff --git a/onionr/static-data/www/board/theme.css b/onionr/static-data/www/board/theme.css new file mode 100644 index 00000000..766e4407 --- /dev/null +++ b/onionr/static-data/www/board/theme.css @@ -0,0 +1,31 @@ +h1, h2, h3{ + font-family: sans-serif; +} +.hidden{ + display: none; +} +p{ + font-family: sans-serif; +} +#webpassWindow{ + background-color: black; + border: 1px solid black; + border-radius: 5px; + width: 100%; + z-index: 2; + color: white; + text-align: center; +} + +.entry{ + color: red; +} + +#feed{ + margin-left: 2%; + margin-right: 25%; + margin-top: 1em; + border: 2px solid black; + padding: 5px; + min-height: 50px; +} \ No newline at end of file diff --git a/onionr/static-data/www/mail/index.html b/onionr/static-data/www/mail/index.html new file mode 100644 index 00000000..6e8ce037 --- /dev/null +++ b/onionr/static-data/www/mail/index.html @@ -0,0 +1,24 @@ + + + + + + Onionr Mail + + + + + + +
+
+ + Onionr Mail +
+ +
+
+ + + + \ No newline at end of file diff --git a/onionr/static-data/www/mail/mail.css b/onionr/static-data/www/mail/mail.css new file mode 100644 index 00000000..7a807d3e --- /dev/null +++ b/onionr/static-data/www/mail/mail.css @@ -0,0 +1,7 @@ +.threads div{ + padding-top: 1em; +} +.threads div span{ + padding-left: 0.5em; + padding-right: 0.5em; +} \ No newline at end of file diff --git a/onionr/static-data/www/mail/mail.js b/onionr/static-data/www/mail/mail.js new file mode 100644 index 00000000..f392f569 --- /dev/null +++ b/onionr/static-data/www/mail/mail.js @@ -0,0 +1,52 @@ +pms = '' +threadPart = document.getElementById('threads') +function getInbox(){ + for(var i = 0; i < pms.length; i++) { + fetch('/getblockdata/' + pms[i], { + headers: { + "token": webpass + }}) + .then((resp) => resp.json()) // Transform the data into json + .then(function(resp) { + + var entry = document.createElement('div') + + var bHashDisplay = document.createElement('a') + var senderInput = document.createElement('input') + var subjectLine = document.createElement('span') + var dateStr = document.createElement('span') + var humanDate = new Date(0) + humanDate.setUTCSeconds(resp['meta']['time']) + senderInput.value = resp['meta']['signer'] + bHashDisplay.innerText = pms[i - 1].substring(0, 10) + bHashDisplay.setAttribute('hash', pms[i - 1]); + senderInput.readOnly = true + dateStr.innerText = humanDate.toString() + if (resp['metadata']['subject'] === undefined || resp['metadata']['subject'] === null) { + subjectLine.innerText = '()' + } + else{ + subjectLine.innerText = '(' + resp['metadata']['subject'] + ')' + } + //entry.innerHTML = 'sender ' + resp['meta']['signer'] + ' - ' + resp['meta']['time'] + threadPart.appendChild(entry) + entry.appendChild(bHashDisplay) + entry.appendChild(senderInput) + entry.appendChild(subjectLine) + entry.appendChild(dateStr) + + }.bind([pms, i])) + } + +} + +fetch('/getblocksbytype/pm', { + headers: { + "token": webpass + }}) +.then((resp) => resp.text()) // Transform the data into json +.then(function(data) { + pms = data.split(',') + getInbox(pms) + }) + diff --git a/onionr/static-data/www/private/index.html b/onionr/static-data/www/private/index.html new file mode 100644 index 00000000..3e239eb0 --- /dev/null +++ b/onionr/static-data/www/private/index.html @@ -0,0 +1,33 @@ + + + + + + Onionr + + + + + +
+
+

Your node will shutdown. Thank you for using Onionr.

+
+
+ + Onionr Web Control Panel +
+ +

Mail +

Stats

+

Uptime:

+

Stored Blocks:

+

Blocks in queue:

+

Connected nodes:

+ 

+        

+        
+        
+        
+    
+
\ No newline at end of file
diff --git a/onionr/static-data/www/shared/main/stats.js b/onionr/static-data/www/shared/main/stats.js
new file mode 100644
index 00000000..b7d05776
--- /dev/null
+++ b/onionr/static-data/www/shared/main/stats.js
@@ -0,0 +1,32 @@
+/*
+
+    Onionr - P2P Anonymous Storage Network
+
+    This file loads stats to show on the main node web page
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see 
+*/
+uptimeDisplay = document.getElementById('uptime')
+connectedDisplay = document.getElementById('connectedNodes')
+storedBlockDisplay = document.getElementById('storedBlocks')
+queuedBlockDisplay = document.getElementById('blockQueue')
+
+function getStats(){
+    stats = JSON.parse(httpGet('getstats', webpass))
+    uptimeDisplay.innerText = stats['uptime'] + ' seconds'
+    connectedDisplay.innerText = stats['connectedNodes']
+    storedBlockDisplay.innerText = stats['blockCount']
+    queuedBlockDisplay.innerText = stats['blockQueueCount']
+}
+getStats()
\ No newline at end of file
diff --git a/onionr/static-data/www/shared/main/style.css b/onionr/static-data/www/shared/main/style.css
new file mode 100644
index 00000000..69e1a407
--- /dev/null
+++ b/onionr/static-data/www/shared/main/style.css
@@ -0,0 +1,140 @@
+body{
+    background-color: #2c2b3f;
+    color: white;
+}
+
+a, a:visited{
+    color: white;
+}
+.center{
+    text-align: center;
+}
+footer{
+    margin-top: 2em;
+    margin-bottom: 0.5em;
+}
+
+body{
+    margin-left: 3em;
+    padding: 1em;
+}
+.onionrMenu{
+    max-width: 25%;
+    margin-left: 2%;
+    margin-right: 10%;
+    font-family: sans-serif;
+}
+.onionrMenu li{
+    list-style-type: none;
+    margin-top: 3px;
+    font-size: 125%;
+}
+.onionrMenu li:hover{
+    color: red;
+}
+.box {
+    display: flex;
+    align-items:center;
+ }
+ .logo{
+    max-width: 25%;
+    vertical-align: middle;
+ }
+ .logoText{
+     font-family: sans-serif;
+     font-size: 2em;
+     margin-top: 1em;
+     margin-left: 1%;
+ }
+ .main{
+     min-height: 500px;
+ }
+
+.content{
+    margin-top: 3em;
+    margin-left: 0%;
+    margin-right: 40%;
+    background-color: white;
+    color: black;
+    padding-right: 5%;
+    padding-left: 3%;
+    padding-bottom: 2em;
+    padding-top: 0.5em;
+    border: 1px solid black;
+    border-radius: 10px;
+    min-height: 300px;
+}
+.content p{
+    text-align: justify;
+}
+.content img{
+    max-width: 35%;
+}
+.content a, .content a:visited{
+    color: black;
+}
+
+.stats{
+    margin-top: 1em;
+    background-color: #0c1049;
+    padding: 5px;
+    margin-right: 45%;
+    font-family: sans-serif;
+}
+.statDesc{
+    background-color: black;
+    padding: 5px;
+    margin-right: 1%;
+    margin-left: -5px;
+}
+
+.stats noscript{
+    color: blue;
+}
+
+.statItem{
+    padding-left: 10px;
+    float: right;
+    margin-right: 5px;
+}
+
+.warn{
+    color: orangered;
+}
+
+@media only screen and (max-width: 640px) {
+    .onionrMenu{
+        margin-left: 0%;
+    }
+    body{
+        margin-left: 0em;
+    }
+    .content{
+        margin-left: 1%;
+        margin-right: 2%;
+    }
+    .content img{
+        max-width: 85%;
+    }
+    .stats{
+        margin-right: 1%;
+    }
+    .statItem{
+        float: initial;
+        display: block;
+    }
+}
+
+/*https://stackoverflow.com/a/16778646*/
+.overlay {
+    visibility: hidden;
+    position: absolute;
+    left: 0px;
+    top: 0px;
+    width:100%;
+    opacity: 0.9;
+    height:100%;
+    text-align:center;
+    z-index: 1000;
+    background-color: black;
+ }
diff --git a/onionr/static-data/www/shared/misc.js b/onionr/static-data/www/shared/misc.js
new file mode 100644
index 00000000..c9b3790e
--- /dev/null
+++ b/onionr/static-data/www/shared/misc.js
@@ -0,0 +1,44 @@
+webpass = document.location.hash.replace('#', '')
+nowebpass = false
+if (typeof webpass == "undefined"){
+    webpass = localStorage['webpass']
+}
+else{
+    localStorage['webpass'] = webpass
+    //document.location.hash = ''
+}
+if (typeof webpass == "undefined" || webpass == ""){
+    alert('Web password was not found in memory or URL')
+    nowebpass = true
+}
+
+function httpGet(theUrl) {
+    var xmlHttp = new XMLHttpRequest()
+    xmlHttp.open( "GET", theUrl, false ) // false for synchronous request
+    xmlHttp.setRequestHeader('token', webpass)
+    xmlHttp.send( null )
+    if (xmlHttp.status == 200){
+        return xmlHttp.responseText
+    }
+    else{
+        return ""
+    }
+}
+function overlay(overlayID) {
+    el = document.getElementById(overlayID)
+   el.style.visibility = (el.style.visibility == "visible") ? "hidden" : "visible"
+ }
+
+var passLinks = document.getElementsByClassName("idLink")
+ for(var i = 0; i < passLinks.length; i++) {
+    passLinks[i].href += '#' + webpass
+ }
+ 
+var refreshLinks = document.getElementsByClassName("refresh")
+
+for(var i = 0; i < refreshLinks.length; i++) {
+    //Can't use .reload because of webpass
+    refreshLinks[i].onclick = function(){
+        location.reload()
+    }
+}
diff --git a/onionr/static-data/www/shared/onionr-icon.png b/onionr/static-data/www/shared/onionr-icon.png
new file mode 100644
index 00000000..6662210d
Binary files /dev/null and b/onionr/static-data/www/shared/onionr-icon.png differ
diff --git a/onionr/static-data/www/shared/onionrblocks.js b/onionr/static-data/www/shared/onionrblocks.js
new file mode 100644
index 00000000..6be0210d
--- /dev/null
+++ b/onionr/static-data/www/shared/onionrblocks.js
@@ -0,0 +1,7 @@
+class Block {
+    constructor(hash, raw) {
+        this.hash = hash;
+        this.raw = raw;
+    }
+}
+   
\ No newline at end of file
diff --git a/onionr/static-data/www/shared/panel.js b/onionr/static-data/www/shared/panel.js
new file mode 100644
index 00000000..665904e8
--- /dev/null
+++ b/onionr/static-data/www/shared/panel.js
@@ -0,0 +1,12 @@
+shutdownBtn = document.getElementById('shutdownNode')
+refreshStatsBtn = document.getElementById('refreshStats')
+shutdownBtn.onclick = function(){
+    if (! nowebpass){
+        httpGet('shutdownclean')
+        overlay('shutdownNotice')
+    }
+}
+
+refreshStatsBtn.onclick = function(){
+    getStats()
+}
\ No newline at end of file
diff --git a/onionr/static-data/www/ui/dist/js/main.js b/onionr/static-data/www/ui/dist/js/main.js
index 6fc7c3d1..0ddf141e 100644
--- a/onionr/static-data/www/ui/dist/js/main.js
+++ b/onionr/static-data/www/ui/dist/js/main.js
@@ -704,7 +704,7 @@ if(tt !== null && tt !== undefined) {
 if(getWebPassword() === null) {
     var password = "";
     while(password.length != 64) {
-        password = prompt("Please enter the web password (run `./RUN-LINUX.sh --get-password`)");
+        password = prompt("Please enter the web password (run `./RUN-LINUX.sh --details`)");
     }
 
     setWebPassword(password);
diff --git a/onionr/storagecounter.py b/onionr/storagecounter.py
index 4468dacc..fc1a9d6b 100644
--- a/onionr/storagecounter.py
+++ b/onionr/storagecounter.py
@@ -42,7 +42,14 @@ class StorageCounter:
                 retData = int(dataFile.read())
         except FileNotFoundError:
             pass
+        except ValueError:
+            pass # Possibly happens when the file is empty
         return retData
+    
+    def getPercent(self):
+        '''Return percent (decimal/float) of disk space we're using'''
+        amount = self.getAmount()
+        return round(amount / self._core.config.get('allocations.disk'), 2)
 
     def addBytes(self, amount):
         '''Record that we are now using more disk space, unless doing so would exceed configured max'''
diff --git a/start-daemon.sh b/start-daemon.sh
new file mode 100755
index 00000000..1b713100
--- /dev/null
+++ b/start-daemon.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/bash
+cd "$(dirname "$0")"
+echo "starting Onionr daemon..."
+echo "run onionr.sh stop to stop the daemon, or onionr.sh start to get output"
+nohup ./onionr.sh start & disown > /dev/null