work on lan

master
Kevin Froman 2020-04-06 08:51:20 -05:00
parent a5983d32a4
commit a52465a54f
13 changed files with 151 additions and 33 deletions

View File

@ -1,3 +1,6 @@
# sse # sse
This folder contains a wrapper for handling server sent event loops This folder contains a wrapper for handling server sent event loops

View File

@ -36,5 +36,4 @@ class LANManager:
def start(self): def start(self):
Thread(target=learn_services, args=[self.too_many.get(Client)], daemon=True).start() Thread(target=learn_services, args=[self.too_many.get(Client)], daemon=True).start()
Thread(target=advertise_service, daemon=True).start() Thread(target=advertise_service, daemon=True).start()
Thread(target=self.too_many.get(Client, (self.peers,)).start, daemon=True).start()

View File

@ -4,9 +4,15 @@ LAN transport client thread
""" """
from typing import List from typing import List
import watchdog
from requests.exceptions import ConnectionError
from onionrcrypto.cryptoutils.randomshuffle import random_shuffle from onionrcrypto.cryptoutils.randomshuffle import random_shuffle
from utils.bettersleep import better_sleep from utils.bettersleep import better_sleep
from onionrutils.basicrequests import do_post_request, do_get_request from onionrutils.basicrequests import do_post_request, do_get_request
from threading import Thread
from onionrblocks import BlockList
from onionrblocks.blockimporter import import_block_from_data
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@ -29,17 +35,46 @@ class Client:
self.lookup_time = {} self.lookup_time = {}
self.poll_delay = 10 self.poll_delay = 10
def get_lookup_time(self, peer): def get_lookup_time(self, peer):
try: try:
return self.lookup_time[peer] return self.lookup_time[peer]
except KeyError: except KeyError:
return 0 return 0
def start(self): def peer_work(self, peer):
port = 1024
self.peers.append(peer)
for port in range(port, 65535):
print(port)
try:
if do_get_request(f'http://{peer}:{port}/ping', proxyType='lan', ignoreAPI=True, connect_timeout=0.3) == 'onionr!':
port = port
print(f'{peer}:{port} found')
break
except (AttributeError, ConnectionError):
pass
else:
self.peers.remove(peer)
return
self.peers.append(peer)
while True: while True:
self.peers = random_shuffle(self.peers) block_list = self._too_many.get(BlockList).get()
last_time = self.get_lookup_time(peer)
new_blocks = set('\n'.join(do_get_request(f'http://{peer}:{port}/blist/{last_time}', proxyType='lan', ignoreAPI=True))) ^ set(block_list)
for bl in new_blocks:
import_block_from_data(
do_get_request(
f'http://{peer}:{port}/get/{bl}', proxyType='lan', ignoreAPI=True))
better_sleep(10)
self.peers.remove(peer)
def connect_peer(self, peer):
better_sleep(self.poll_delay) if peer in self.peers:
return
print(f'connecting to {peer}')
Thread(target=self.peer_work, args=[peer], daemon=True).start()

View File

@ -9,7 +9,7 @@ from typing import List
from ipaddress import ip_address from ipaddress import ip_address
from socket import SHUT_RDWR from socket import SHUT_RDWR
from .getip import lan_ips from .getip import lan_ips, best_ip
from utils.bettersleep import better_sleep from utils.bettersleep import better_sleep
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
@ -52,17 +52,17 @@ def learn_services(lan_client):
if 'onionr' not in service_ips: if 'onionr' not in service_ips:
continue continue
service_ips = service_ips.replace('onionr-', '').split('-') service_ips = service_ips.replace('onionr-', '').split('-')
print(service_ips)
port = 0 port = 0
for service in service_ips: for service in service_ips:
try: try:
ip_address(service) ip_address(service)
if not ip_address(service).is_private: raise ValueError if not ip_address(service).is_private: raise ValueError
if service in lan_ips: raise ValueError if service in lan_ips: raise ValueError
if service in lan_client.peers: raise ValueError
except ValueError: except ValueError:
service_ips.remove(service) pass
p = list(lan_client.peers) else:
lan_client.peers = list(set(service_ips + p)) lan_client.connect_peer(service)
def advertise_service(specific_ips=None): def advertise_service(specific_ips=None):
@ -71,10 +71,8 @@ def advertise_service(specific_ips=None):
# for all packets sent, after three hops on the network the packet will not # for all packets sent, after three hops on the network the packet will not
# be re-sent/broadcast (see https://www.tldp.org/HOWTO/Multicast-HOWTO-6.html) # be re-sent/broadcast (see https://www.tldp.org/HOWTO/Multicast-HOWTO-6.html)
MULTICAST_TTL = 3 MULTICAST_TTL = 3
if specific_ips is None:
ips = '-'.join(lan_ips) ips = best_ip
else:
ips = '-'.join(specific_ips)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, MULTICAST_TTL) sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, MULTICAST_TTL)

View File

@ -6,6 +6,7 @@ from gevent.pywsgi import WSGIServer
from flask import Flask from flask import Flask
from flask import Response from flask import Response
from flask import request from flask import request
from flask import abort
from onionrblocks.onionrblockapi import Block from onionrblocks.onionrblockapi import Block
from httpapi.fdsafehandler import FDSafeHandler from httpapi.fdsafehandler import FDSafeHandler
@ -15,6 +16,7 @@ from coredb.blockmetadb import get_block_list
from lan.getip import best_ip from lan.getip import best_ip
from onionrutils import stringvalidators from onionrutils import stringvalidators
from httpapi.miscpublicapi.upload import accept_upload from httpapi.miscpublicapi.upload import accept_upload
import logger
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@ -41,6 +43,13 @@ class LANServer:
self.host = best_ip self.host = best_ip
self.port = None self.port = None
@app.before_request
def dns_rebinding_prevention():
if request.host != f'{self.host}:{self.port}':
logger.warn('Potential DNS rebinding attack on LAN server:')
logger.warn(f'Hostname {request.host} was used instead of {self.host}:{self.port}')
abort(403)
@app.route('/blist/<time>') @app.route('/blist/<time>')
def get_block_list_for_lan(time): def get_block_list_for_lan(time):
return Response('\n'.join(get_block_list(dateRec=time))) return Response('\n'.join(get_block_list(dateRec=time)))
@ -54,7 +63,7 @@ class LANServer:
@app.route("/ping") @app.route("/ping")
def ping(): def ping():
return Response("pong!") return Response("onionr!")
@app.route('/upload', methods=['POST']) @app.route('/upload', methods=['POST'])
def upload_endpoint(): def upload_endpoint():
@ -65,5 +74,5 @@ class LANServer:
self.app, log=None, self.app, log=None,
handler_class=FDSafeHandler) handler_class=FDSafeHandler)
self.port = self.server.server_port self.port = self.server.server_port
logger.info(f'Serving to LAN on {self.host}:{self.port}', terminal=True)
self.server.serve_forever() self.server.serve_forever()

View File

@ -1,5 +1,5 @@
from . import insert from . import insert
from .insert import time_insert from .insert import time_insert
from .blocklist import BlockList
insert = insert.insert_block insert = insert.insert_block
time_insert = time_insert time_insert = time_insert

View File

@ -0,0 +1,41 @@
from threading import Thread
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
from utils.identifyhome import identify_home
from coredb.dbfiles import block_meta_db
from coredb.blockmetadb import get_block_list, get_blocks_by_type
from onionrutils.epoch import get_epoch
class BlockList:
def __init__(self, auto_refresh=True, block_type=''):
self.block_type = block_type
self.refresh_db()
self.check_time = get_epoch()
class Refresher(FileSystemEventHandler):
@staticmethod
def on_modified(event):
if event.src_path != block_meta_db:
return
self.refresh_db()
if auto_refresh:
def auto_refresher():
observer = Observer()
observer.schedule(Refresher(), identify_home(), recursive=False)
observer.start()
while observer.is_alive():
# call import func with timeout
observer.join(120)
Thread(target=auto_refresher, daemon=True).start()
def get(self):
return self.block_list
def refresh_db(self):
self.check_time = get_epoch()
if not self.block_type:
self.block_list = get_block_list()
else:
self.block_list = get_blocks_by_type(self.block_type)

View File

@ -57,7 +57,7 @@ def do_post_request(url, data={}, port=0, proxyType='tor', max_size=10000, conte
return retData return retData
def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880): def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880, connect_timeout=15):
''' '''
Do a get request through a local tor or i2p instance Do a get request through a local tor or i2p instance
''' '''
@ -72,7 +72,7 @@ def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=
elif proxyType == 'lan': elif proxyType == 'lan':
address = urlparse(url).hostname address = urlparse(url).hostname
if IPv4Address(address).is_private and not IPv4Address(address).is_loopback: if IPv4Address(address).is_private and not IPv4Address(address).is_loopback:
proxies = {} proxies = None
else: else:
return return
else: else:
@ -80,8 +80,9 @@ def do_get_request(url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=
headers = {'User-Agent': 'PyOnionr', 'Connection':'close'} headers = {'User-Agent': 'PyOnionr', 'Connection':'close'}
response_headers = dict() response_headers = dict()
try: try:
proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)} if not proxies is None:
r = streamedrequests.get(url, request_headers=headers, allow_redirects=False, proxy=proxies, connect_timeout=15, stream_timeout=120, max_size=max_size) proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
r = streamedrequests.get(url, request_headers=headers, allow_redirects=False, proxy=proxies, connect_timeout=connect_timeout, stream_timeout=120, max_size=max_size)
# Check server is using same API version as us # Check server is using same API version as us
if not ignoreAPI: if not ignoreAPI:
try: try:

View File

@ -2,6 +2,7 @@
Return a useful tuple of (metadata (header), meta, and data) by accepting raw block data Return a useful tuple of (metadata (header), meta, and data) by accepting raw block data
""" """
from json import JSONDecodeError
import ujson as json import ujson as json
from onionrutils import bytesconverter from onionrutils import bytesconverter
@ -37,7 +38,7 @@ def get_block_metadata_from_data(block_data):
try: try:
metadata = json.loads(bytesconverter.bytes_to_str(block_data[:block_data.find(b'\n')])) metadata = json.loads(bytesconverter.bytes_to_str(block_data[:block_data.find(b'\n')]))
except json.decoder.JSONDecodeError: except JSONDecodeError:
pass pass
else: else:
data = block_data[block_data.find(b'\n'):] data = block_data[block_data.find(b'\n'):]

View File

@ -12,7 +12,7 @@ def test_lan_server(testmanager):
start_time = get_epoch() start_time = get_epoch()
for i in range(1024, 65536): for i in range(1024, 65536):
try: try:
if requests.get(f"http://{best_ip}:{i}/ping").text == 'pong!': if requests.get(f"http://{best_ip}:{i}/ping").text == 'onionr!':
bl = insert('test data') bl = insert('test data')
sleep(10) sleep(10)
bl2 = insert('test data2') bl2 = insert('test data2')

View File

@ -7,16 +7,16 @@
}, },
"general": { "general": {
"announce_node": true, "announce_node": true,
"dev_mode": false, "dev_mode": true,
"display_header": true, "display_header": true,
"ephemeral_tunnels": false, "ephemeral_tunnels": false,
"hide_created_blocks": true, "hide_created_blocks": true,
"insert_deniable_blocks": true, "insert_deniable_blocks": false,
"max_block_age": 2678400, "max_block_age": 2678400,
"minimum_block_pow": 5, "minimum_block_pow": 1,
"minimum_send_pow": 5, "minimum_send_pow": 1,
"public_key": "", "public_key": "",
"random_bind_ip": true, "random_bind_ip": false,
"security_level": 0, "security_level": 0,
"show_notifications": true, "show_notifications": true,
"store_plaintext_blocks": true, "store_plaintext_blocks": true,
@ -30,12 +30,12 @@
}, },
"file": { "file": {
"output": true, "output": true,
"remove_on_exit": true "remove_on_exit": false
}, },
"verbosity": "default" "verbosity": "default"
}, },
"onboarding": { "onboarding": {
"done": false "done": true
}, },
"peers": { "peers": {
"max_connect": 1000, "max_connect": 1000,
@ -68,7 +68,7 @@
"transports": { "transports": {
"lan": true, "lan": true,
"sneakernet": true, "sneakernet": true,
"tor": true "tor": false
}, },
"ui": { "ui": {
"theme": "dark" "theme": "dark"

View File

@ -1 +1 @@
1585984468 1586149428

View File

@ -0,0 +1,31 @@
#!/usr/bin/env python3
import sys, os
sys.path.append(".")
sys.path.append("src/")
import uuid
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
print("Test directory:", TEST_DIR)
os.environ["ONIONR_HOME"] = TEST_DIR
import unittest
from utils import identifyhome, createdirs, bettersleep
from onionrsetup import setup_config
from onionrblocks import BlockList, insert
createdirs.create_dirs()
setup_config()
class TestBlockList(unittest.TestCase):
def test_block_list(self):
block_list = BlockList()
self.assertEqual(len(block_list.get()), 0)
bl = insert('test')
bettersleep.better_sleep(0.8)
self.assertIn(bl, block_list.get())
bl2 = insert('test2')
bettersleep.better_sleep(0.8)
self.assertIn(bl2, block_list.get())
self.assertIn(bl, block_list.get())
unittest.main()