Formatting improvements in httpapis

master
Kevin Froman 2020-08-26 08:25:43 +00:00
parent 3422ca43ff
commit 14f2d03ebf
14 changed files with 88 additions and 60 deletions

View File

@ -1,5 +1,7 @@
from gevent.pywsgi import WSGIServer, WSGIHandler from gevent.pywsgi import WSGIServer, WSGIHandler
from gevent import Timeout from gevent import Timeout
class FDSafeHandler(WSGIHandler): class FDSafeHandler(WSGIHandler):
'''Our WSGI handler. Doesn't do much non-default except timeouts''' '''Our WSGI handler. Doesn't do much non-default except timeouts'''
def handle(self): def handle(self):

View File

@ -78,7 +78,7 @@ def client_api_insert_block():
pass pass
try: try:
# The setting in the UI is for if forward secrecy is enabled, not disabled # Setting in the mail UI is for if forward secrecy is *enabled*
disable_forward_secrecy = not insert_data['forward'] disable_forward_secrecy = not insert_data['forward']
except KeyError: except KeyError:
disable_forward_secrecy = False disable_forward_secrecy = False

View File

@ -9,7 +9,6 @@ import logger
from etc import onionrvalues from etc import onionrvalues
from onionrutils import stringvalidators, bytesconverter from onionrutils import stringvalidators, bytesconverter
import filepaths import filepaths
from communicator import OnionrCommunicatorDaemon
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by

View File

@ -1,9 +1,13 @@
''' """Onionr - Private P2P Communication.
Onionr - Private P2P Communication
Misc public API endpoints too small to need their own file and that need access to the public api inst Misc public API endpoints too small to need their own file
''' and that need access to the public api inst
''' """
from flask import Response, Blueprint, request, send_from_directory, abort, g
from . import getblocks, upload, announce
from coredb import keydb
import config
"""
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
@ -16,11 +20,9 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' """
from flask import Response, Blueprint, request, send_from_directory, abort, g
from . import getblocks, upload, announce
from coredb import keydb
import config
class PublicEndpoints: class PublicEndpoints:
def __init__(self, public_api): def __init__(self, public_api):
@ -39,7 +41,8 @@ class PublicEndpoints:
@public_endpoints_bp.route('/getblocklist') @public_endpoints_bp.route('/getblocklist')
def get_block_list(): def get_block_list():
'''Get a list of blocks, optionally filtered by epoch time stamp, excluding those hidden''' """Get a list of blocks, optionally filtered by epoch time stamp,
excluding those hidden"""
return getblocks.get_public_block_list(public_api, request) return getblocks.get_public_block_list(public_api, request)
@public_endpoints_bp.route('/getdata/<name>') @public_endpoints_bp.route('/getdata/<name>')
@ -68,14 +71,15 @@ class PublicEndpoints:
@public_endpoints_bp.route('/announce', methods=['post']) @public_endpoints_bp.route('/announce', methods=['post'])
def accept_announce(): def accept_announce():
'''Accept announcements with pow token to prevent spam''' """Accept announcements with pow token to prevent spam"""
g.shared_state = public_api._too_many g.shared_state = public_api._too_many
resp = announce.handle_announce(request) resp = announce.handle_announce(request)
return resp return resp
@public_endpoints_bp.route('/upload', methods=['post']) @public_endpoints_bp.route('/upload', methods=['post'])
def upload_endpoint(): def upload_endpoint():
'''Accept file uploads. In the future this will be done more often than on creation """Accept file uploads.
In the future this will be done more often than on creation
to speed up block sync to speed up block sync
''' """
return upload.accept_upload(request) return upload.accept_upload(request)

View File

@ -45,11 +45,14 @@ def get_block_data(publicAPI, data):
"""data is the block hash in hex""" """data is the block hash in hex"""
resp = '' resp = ''
if stringvalidators.validate_hash(data): if stringvalidators.validate_hash(data):
if not config.get('general.hide_created_blocks', True) or data not in publicAPI.hideBlocks: if not config.get('general.hide_created_blocks', True) \
or data not in publicAPI.hideBlocks:
if data in publicAPI._too_many.get(BlockList).get(): if data in publicAPI._too_many.get(BlockList).get():
block = apiutils.GetBlockData().get_block_data(data, raw=True, decrypt=False) block = apiutils.GetBlockData().get_block_data(
data, raw=True, decrypt=False)
try: try:
block = block.encode('utf-8') # Encode in case data is binary # Encode in case data is binary
block = block.encode('utf-8')
except AttributeError: except AttributeError:
if len(block) == 0: if len(block) == 0:
abort(404) abort(404)

View File

@ -5,7 +5,6 @@ Accept block uploads to the public API server
import sys import sys
from gevent import spawn from gevent import spawn
from gevent import threading
from flask import Response from flask import Response
from flask import abort from flask import abort
from flask import g from flask import g
@ -43,7 +42,7 @@ def accept_upload(request):
if g.too_many.get_by_string("DeadSimpleKV").get('onlinePeers'): if g.too_many.get_by_string("DeadSimpleKV").get('onlinePeers'):
spawn( spawn(
localcommand.local_command, localcommand.local_command,
f'/daemon-event/upload_event', '/daemon-event/upload_event',
post=True, post=True,
is_json=True, is_json=True,
post_data={'block': b_hash} post_data={'block': b_hash}

View File

@ -1,6 +1,7 @@
"""Onionr - Private P2P Communication. """Onionr - Private P2P Communication.
Process incoming requests to the client api server to validate they are legitimate Process incoming requests to the client api server to validate
that they are legitimate and not DNSR/XSRF or other local adversary
""" """
import hmac import hmac
from flask import Blueprint, request, abort, g from flask import Blueprint, request, abort, g
@ -21,17 +22,23 @@ from . import pluginwhitelist
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
""" """
# Be extremely mindful of this. These are endpoints available without a password # Be extremely mindful of this.
whitelist_endpoints = ['www', 'staticfiles.homedata', 'staticfiles.sharedContent', # These are endpoints available without a password
'staticfiles.friends', 'staticfiles.friendsindex', 'siteapi.site', 'siteapi.siteFile', 'staticfiles.onionrhome', whitelist_endpoints = [
'www', 'staticfiles.homedata',
'staticfiles.sharedContent',
'staticfiles.friends', 'staticfiles.friendsindex', 'siteapi.site',
'siteapi.siteFile', 'staticfiles.onionrhome',
'themes.getTheme', 'staticfiles.onboarding', 'staticfiles.onboardingIndex'] 'themes.getTheme', 'staticfiles.onboarding', 'staticfiles.onboardingIndex']
class ClientAPISecurity: class ClientAPISecurity:
def __init__(self, client_api): def __init__(self, client_api):
client_api_security_bp = Blueprint('clientapisecurity', __name__) client_api_security_bp = Blueprint('clientapisecurity', __name__)
self.client_api_security_bp = client_api_security_bp self.client_api_security_bp = client_api_security_bp
self.client_api = client_api self.client_api = client_api
pluginwhitelist.load_plugin_security_whitelist_endpoints(whitelist_endpoints) pluginwhitelist.load_plugin_security_whitelist_endpoints(
whitelist_endpoints)
@client_api_security_bp.before_app_request @client_api_security_bp.before_app_request
def validate_request(): def validate_request():
@ -48,14 +55,18 @@ class ClientAPISecurity:
if request.endpoint in whitelist_endpoints: if request.endpoint in whitelist_endpoints:
return return
if request.path.startswith('/site/'): return if request.path.startswith('/site/'):
return
try: try:
if not hmac.compare_digest(request.headers['token'], client_api.clientToken): if not hmac.compare_digest(
if not hmac.compare_digest(request.form['token'], client_api.clientToken): request.headers['token'], client_api.clientToken):
if not hmac.compare_digest(
request.form['token'], client_api.clientToken):
abort(403) abort(403)
except KeyError: except KeyError:
if not hmac.compare_digest(request.form['token'], client_api.clientToken): if not hmac.compare_digest(
request.form['token'], client_api.clientToken):
abort(403) abort(403)
@client_api_security_bp.after_app_request @client_api_security_bp.after_app_request
@ -63,7 +74,9 @@ class ClientAPISecurity:
# Security headers # Security headers
resp = httpheaders.set_default_onionr_http_headers(resp) resp = httpheaders.set_default_onionr_http_headers(resp)
if request.endpoint in ('siteapi.site', 'siteapi.siteFile'): if request.endpoint in ('siteapi.site', 'siteapi.siteFile'):
resp.headers['Content-Security-Policy'] = "default-src 'none'; style-src 'self' data: 'unsafe-inline'; img-src 'self' data:; media-src 'self' data:" resp.headers['Content-Security-Policy'] = \
"default-src 'none'; style-src 'self' data: 'unsafe-inline'; img-src 'self' data:; media-src 'self' data:" # noqa
else: else:
resp.headers['Content-Security-Policy'] = "default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; media-src 'self'; frame-src 'none'; font-src 'self'; connect-src 'self'" resp.headers['Content-Security-Policy'] = \
"default-src 'none'; script-src 'self'; object-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; media-src 'self'; frame-src 'none'; font-src 'self'; connect-src 'self'" # noqa
return resp return resp

View File

@ -59,7 +59,8 @@ class LANAPISecurity:
'Clear-Site-Data', 'Referrer-Policy') 'Clear-Site-Data', 'Referrer-Policy')
try: try:
if g.is_onionr_client: if g.is_onionr_client:
for header in NON_NETWORK_HEADERS: del resp.headers[header] for header in NON_NETWORK_HEADERS:
del resp.headers[header]
except AttributeError: except AttributeError:
abort(403) abort(403)
lan_client.lastRequest = epoch.get_rounded_epoch(roundS=5) lan_client.lastRequest = epoch.get_rounded_epoch(roundS=5)

View File

@ -64,13 +64,15 @@ class PublicAPISecurity:
# Network API version # Network API version
resp.headers['X-API'] = public_api.API_VERSION resp.headers['X-API'] = public_api.API_VERSION
# Delete some HTTP headers for Onionr user agents # Delete some HTTP headers for Onionr user agents
NON_NETWORK_HEADERS = ('Content-Security-Policy', 'X-Frame-Options', NON_NETWORK_HEADERS = (
'Content-Security-Policy', 'X-Frame-Options',
'X-Content-Type-Options', 'Feature-Policy', 'X-Content-Type-Options', 'Feature-Policy',
'Clear-Site-Data', 'Referrer-Policy') 'Clear-Site-Data', 'Referrer-Policy')
try: try:
if g.is_onionr_client: if g.is_onionr_client:
for header in NON_NETWORK_HEADERS: del resp.headers[header] for header in NON_NETWORK_HEADERS:
del resp.headers[header]
except AttributeError: except AttributeError:
abort(403) abort(403)

View File

@ -31,7 +31,6 @@ class LANManager:
self.too_many = too_many self.too_many = too_many
self.peers: "exploded IP Address string" = [] self.peers: "exploded IP Address string" = []
def start(self): def start(self):
Thread(target=learn_services, daemon=True).start() Thread(target=learn_services, daemon=True).start()
Thread(target=advertise_service, daemon=True).start() Thread(target=advertise_service, daemon=True).start()

View File

@ -7,7 +7,6 @@ import requests
from typing import Set from typing import Set
from onionrtypes import LANIP from onionrtypes import LANIP
from utils.bettersleep import better_sleep
import logger import logger
from coredb.blockmetadb import get_block_list from coredb.blockmetadb import get_block_list
from onionrblocks.blockimporter import import_block_from_data from onionrblocks.blockimporter import import_block_from_data
@ -38,8 +37,9 @@ def _lan_work(peer: LANIP):
blocks = requests.get(url + 'blist/0').text.splitlines() blocks = requests.get(url + 'blist/0').text.splitlines()
for block in blocks: for block in blocks:
if block not in our_blocks: if block not in our_blocks:
import_block_from_data(requests.get(url + f'get/{block}', stream=True).raw.read(6000000)) import_block_from_data(
requests.get(
url + f'get/{block}', stream=True).raw.read(6000000))
for port in ports: for port in ports:
try: try:

View File

@ -4,10 +4,7 @@ Discover and publish private-network
""" """
import socket import socket
import struct import struct
from typing import TYPE_CHECKING
from typing import List
from ipaddress import ip_address from ipaddress import ip_address
from socket import SHUT_RDWR
from .getip import lan_ips, best_ip from .getip import lan_ips, best_ip
from utils.bettersleep import better_sleep from utils.bettersleep import better_sleep
@ -32,7 +29,6 @@ IS_ALL_GROUPS = True
ANNOUNCE_LOOP_SLEEP = 30 ANNOUNCE_LOOP_SLEEP = 30
def learn_services(): def learn_services():
"""Take a list to infintely add lan service info to.""" """Take a list to infintely add lan service info to."""
@ -54,12 +50,13 @@ def learn_services():
continue continue
service_ips = service_ips.replace('onionr-', '').split('-') service_ips = service_ips.replace('onionr-', '').split('-')
port = 0
for service in service_ips: for service in service_ips:
try: try:
ip_address(service) ip_address(service)
if not ip_address(service).is_private: raise ValueError if not ip_address(service).is_private:
if service in lan_ips: raise ValueError raise ValueError
if service in lan_ips:
raise ValueError
except ValueError: except ValueError:
pass pass
else: else:
@ -70,7 +67,8 @@ def advertise_service(specific_ips=None):
# regarding socket.IP_MULTICAST_TTL # regarding socket.IP_MULTICAST_TTL
# --------------------------------- # ---------------------------------
# for all packets sent, after three hops on the network the packet will not # for all packets sent, after three hops on the network the packet will not
# be re-sent/broadcast (see https://www.tldp.org/HOWTO/Multicast-HOWTO-6.html) # be re-sent/broadcast
# (see https://www.tldp.org/HOWTO/Multicast-HOWTO-6.html)
MULTICAST_TTL = 3 MULTICAST_TTL = 3
ips = best_ip ips = best_ip

View File

@ -39,6 +39,7 @@ from utils.bettersleep import better_sleep
ports = range(1337, 1340) ports = range(1337, 1340)
_start_time = time.time() _start_time = time.time()
class LANServer: class LANServer:
def __init__(self, shared_state): def __init__(self, shared_state):
app = Flask(__name__) app = Flask(__name__)
@ -51,12 +52,14 @@ class LANServer:
@app.before_request @app.before_request
def dns_rebinding_prevention(): def dns_rebinding_prevention():
if request.remote_addr in lan_ips or ipaddress.ip_address(request.remote_addr).is_loopback: if request.remote_addr in lan_ips or \
ipaddress.ip_address(request.remote_addr).is_loopback:
if time.time() - _start_time > 600: if time.time() - _start_time > 600:
abort(403) abort(403)
if request.host != f'{self.host}:{self.port}': if request.host != f'{self.host}:{self.port}':
logger.warn('Potential DNS rebinding attack on LAN server:') logger.warn('Potential DNS rebinding attack on LAN server:')
logger.warn(f'Hostname {request.host} was used instead of {self.host}:{self.port}') logger.warn(
f'Hostname {request.host} was used instead of {self.host}:{self.port}') # noqa
abort(403) abort(403)
@app.route('/blist/<time>') @app.route('/blist/<time>')
@ -82,9 +85,13 @@ class LANServer:
def _show_lan_bind(port): def _show_lan_bind(port):
better_sleep(1) better_sleep(1)
if self.server.started and port == self.server.server_port: if self.server.started and port == self.server.server_port:
logger.info(f'Serving to LAN on {self.host}:{self.port}', terminal=True) logger.info(
f'Serving to LAN on {self.host}:{self.port}',
terminal=True)
if self.host == "": if self.host == "":
logger.info("Not binding to LAN due to no private network configured.", terminal=True) logger.info(
"Not binding to LAN due to no private network configured.",
terminal=True)
return return
for i in ports: for i in ports:
self.server = WSGIServer((self.host, i), self.server = WSGIServer((self.host, i),
@ -99,5 +106,6 @@ class LANServer:
else: else:
break break
else: else:
logger.warn("Could not bind to any LAN ports " + str(min(ports)) + "-" + str(max(ports)), terminal=True) logger.warn("Could not bind to any LAN ports " +
str(min(ports)) + "-" + str(max(ports)), terminal=True)
return return