renamed onionr dir and bugfixes/linting progress

src/httpapi/miscpublicapi/__init__.py

@@ -0,0 +1,6 @@
+from . import announce, upload, getblocks, endpoints
+
+announce = announce.handle_announce # endpoint handler for accepting peer announcements
+upload = upload.accept_upload # endpoint handler for accepting public uploads
+public_block_list = getblocks.get_public_block_list # endpoint handler for getting block lists
+public_get_block_data = getblocks.get_block_data # endpoint handler for responding to peers requests for block data

src/httpapi/miscpublicapi/announce.py

@@ -0,0 +1,76 @@
+'''
+    Onionr - Private P2P Communication
+
+    Handle announcements to the public API server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import base64
+from flask import Response, g
+import deadsimplekv
+import logger
+from etc import onionrvalues
+from onionrutils import stringvalidators, bytesconverter
+from utils import gettransports
+import onionrcrypto as crypto, filepaths
+from communicator import OnionrCommunicatorDaemon
+def handle_announce(request):
+    '''
+    accept announcement posts, validating POW
+    clientAPI should be an instance of the clientAPI server running, request is a instance of a flask request
+    '''
+    resp = 'failure'
+    powHash = ''
+    randomData = ''
+    newNode = ''
+
+    try:
+        newNode = request.form['node'].encode()
+    except KeyError:
+        logger.warn('No node specified for upload')
+        pass
+    else:
+        try:
+            randomData = request.form['random']
+            randomData = base64.b64decode(randomData)
+        except KeyError:
+            logger.warn('No random data specified for upload')
+        else:
+            nodes = newNode + bytesconverter.str_to_bytes(gettransports.get()[0])
+            nodes = crypto.hashers.blake2b_hash(nodes)
+            powHash = crypto.hashers.blake2b_hash(randomData + nodes)
+            try:
+                powHash = powHash.decode()
+            except AttributeError:
+                pass
+            if powHash.startswith('0' * onionrvalues.ANNOUNCE_POW):
+                newNode = bytesconverter.bytes_to_str(newNode)
+                announce_queue = deadsimplekv.DeadSimpleKV(filepaths.announce_cache)
+                announce_queue_list = announce_queue.get('new_peers')
+                if announce_queue_list is None:
+                    announce_queue_list = []
+
+                if stringvalidators.validate_transport(newNode) and not newNode in announce_queue_list:
+                    #clientAPI.onionrInst.communicatorInst.newPeers.append(newNode)
+                    g.shared_state.get(OnionrCommunicatorDaemon).newPeers.append(newNode)
+                    announce_queue.put('new_peers', announce_queue_list.append(newNode))
+                    announce_queue.flush()
+                    resp = 'Success'
+            else:
+                logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)
+    resp = Response(resp)
+    if resp == 'failure':
+        return resp, 406
+    return resp

src/httpapi/miscpublicapi/endpoints.py

@@ -0,0 +1,81 @@
+'''
+    Onionr - Private P2P Communication
+
+    Misc public API endpoints too small to need their own file and that need access to the public api inst
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from flask import Response, Blueprint, request, send_from_directory, abort, g
+from . import getblocks, upload, announce
+from coredb import keydb
+import config
+class PublicEndpoints:
+    def __init__(self, public_api):
+
+        public_endpoints_bp = Blueprint('publicendpoints', __name__)
+        self.public_endpoints_bp = public_endpoints_bp
+
+        @public_endpoints_bp.route('/')
+        def banner():
+            # Display a bit of information to people who visit a node address in their browser
+            try:
+                with open('../static-data/index.html', 'r') as html:
+                    resp = Response(html.read(), mimetype='text/html')
+            except FileNotFoundError:
+                resp = Response("")
+            return resp
+
+        @public_endpoints_bp.route('/getblocklist')
+        def get_block_list():
+            '''Get a list of blocks, optionally filtered by epoch time stamp, excluding those hidden'''
+            return getblocks.get_public_block_list(public_api, request)
+
+        @public_endpoints_bp.route('/getdata/<name>')
+        def get_block_data(name):
+            # Share data for a block if we have it and it isn't hidden
+            return getblocks.get_block_data(public_api, name)
+
+        @public_endpoints_bp.route('/www/<path:path>')
+        def www_public(path):
+            # A way to share files directly over your .onion
+            if not config.get("www.public.run", True):
+                abort(403)
+            return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
+
+        @public_endpoints_bp.route('/ping')
+        def ping():
+            # Endpoint to test if nodes are up
+            return Response("pong!")
+
+        @public_endpoints_bp.route('/pex')
+        def peer_exchange():
+            response = ','.join(keydb.listkeys.list_adders(recent=3600))
+            if len(response) == 0:
+                response = ''
+            return Response(response)
+
+        @public_endpoints_bp.route('/announce', methods=['post'])
+        def accept_announce():
+            '''Accept announcements with pow token to prevent spam'''
+            g.shared_state = public_api._too_many
+            resp = announce.handle_announce(request)
+            return resp
+
+        @public_endpoints_bp.route('/upload', methods=['post'])
+        def upload_endpoint():
+            '''Accept file uploads. In the future this will be done more often than on creation 
+            to speed up block sync
+            '''
+            return upload.accept_upload(request)

src/httpapi/miscpublicapi/getblocks.py

@@ -0,0 +1,58 @@
+'''
+    Onionr - Private P2P Communication
+
+    Public endpoints to get block data and lists
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+from flask import Response, abort
+import config
+from onionrutils import bytesconverter, stringvalidators
+from coredb import blockmetadb
+from utils import reconstructhash
+from .. import apiutils
+def get_public_block_list(publicAPI, request):
+    # Provide a list of our blocks, with a date offset
+    dateAdjust = request.args.get('date')
+    bList = blockmetadb.get_block_list(dateRec=dateAdjust)
+    share_list = ''
+    if config.get('general.hide_created_blocks', True):
+        for b in publicAPI.hideBlocks:
+            if b in bList:
+                # Don't share blocks we created if they haven't been *uploaded* yet, makes it harder to find who created a block
+                bList.remove(b)
+    for b in bList:
+        share_list += '%s\n' % (reconstructhash.deconstruct_hash(b),)
+    return Response(share_list)
+
+def get_block_data(publicAPI, data):
+    '''data is the block hash in hex'''
+    resp = ''
+    if stringvalidators.validate_hash(data):
+        if not config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks:
+            if data in blockmetadb.get_block_list():
+                block = apiutils.GetBlockData().get_block_data(data, raw=True)
+                try:
+                    block = block.encode() # Encode in case data is binary
+                except AttributeError:
+                    if len(block) == 0:
+                        abort(404)
+                block = bytesconverter.str_to_bytes(block)
+                resp = block
+    if len(resp) == 0:
+        abort(404)
+        resp = ""
+    # Has to be octet stream, otherwise binary data fails hash check
+    return Response(resp, mimetype='application/octet-stream')

src/httpapi/miscpublicapi/upload.py

@@ -0,0 +1,49 @@
+'''
+    Onionr - Private P2P Communication
+
+    Accept block uploads to the public API server
+'''
+'''
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+'''
+import sys
+from flask import Response, abort
+
+from onionrblocks import blockimporter
+import onionrexceptions, logger
+
+def accept_upload(request):
+    resp = 'failure'
+    data = request.get_data()
+    if sys.getsizeof(data) < 100000000:
+        try:
+            if blockimporter.importBlockFromData(data):
+                resp = 'success'
+            else:
+                resp = 'failure'
+                logger.warn('Error encountered importing uploaded block')
+        except onionrexceptions.BlacklistedBlock:
+            logger.debug('uploaded block is blacklisted')
+            resp = 'failure'
+        except onionrexceptions.InvalidProof:
+            resp = 'proof'
+        except onionrexceptions.DataExists:
+            resp = 'exists'
+    if resp == 'failure':
+        abort(400)
+    elif resp == 'proof':
+        resp = Response(resp, 400)
+    else:
+        resp = Response(resp)
+    return resp