From 1bd0aa9419a397061749c820b3434376a11630be Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Mon, 30 Mar 2020 03:23:59 -0500 Subject: [PATCH] added sneakernet auto importing exportblocks now takes argument --- .vscode/tasks.json | 7 +++ requirements.in | 2 +- requirements.txt | 8 ++- scripts/disable-dev-config.py | 2 +- src/etc/onionrvalues.py | 2 +- src/onionrcommands/daemonlaunch/__init__.py | 3 + src/onionrcommands/exportblocks.py | 26 ++++---- src/onionrutils/importnewblocks.py | 15 ++--- src/runtests/__init__.py | 4 +- src/runtests/sneakernettest.py | 27 +++++++++ src/sneakernet/__init__.py | 54 +++++++++++++++++ .../metadataprocessor/info.json | 5 -- .../default-plugins/metadataprocessor/main.py | 59 ------------------- static-data/default_config.json | 2 +- tests/runtime-result.txt | 2 +- tests/test_default_config_json.py | 2 +- tests/test_onionrvalues.py | 2 +- 17 files changed, 131 insertions(+), 91 deletions(-) create mode 100644 src/runtests/sneakernettest.py create mode 100644 src/sneakernet/__init__.py delete mode 100755 static-data/default-plugins/metadataprocessor/info.json delete mode 100755 static-data/default-plugins/metadataprocessor/main.py diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 8d91b679..26bfed5d 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -46,6 +46,13 @@ "command": "scripts/disable-dev-config.py", "group": "dev", "problemMatcher": [] + }, + { + "label": "Format default config", + "type": "process", + "command": "scripts/pretty-default-config.py", + "group": "dev", + "problemMatcher": [] } ] } \ No newline at end of file diff --git a/requirements.in b/requirements.in index ef7ec644..899df4a8 100644 --- a/requirements.in +++ b/requirements.in @@ -8,9 +8,9 @@ stem==1.8.0 deadsimplekv==0.3.1 unpaddedbase32==0.2.0 streamedrequests==1.0.0 -jinja2==2.11.1 toomanyobjs==1.1.0 niceware==0.2.1 psutil==5.7.0 filenuke==0.0.0 mimcvdf==1.0.0 +watchdog==0.10.2 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 989c5c6f..897b0881 100644 --- a/requirements.txt +++ b/requirements.txt @@ -115,7 +115,7 @@ itsdangerous==1.1.0 \ jinja2==2.11.1 \ --hash=sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250 \ --hash=sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49 \ - # via -r requirements.in, flask + # via flask markupsafe==1.1.1 \ --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \ --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \ @@ -154,6 +154,9 @@ niceware==0.2.1 \ --hash=sha256:0f8b192f2a1e800e068474f6e208be9c7e2857664b33a96f4045340de4e5c69c \ --hash=sha256:cf2dc0e1567d36d067c61b32fed0f1b9c4534ed511f9eeead4ba548d03b5c9eb \ # via -r requirements.in +pathtools==0.1.2 \ + --hash=sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0 \ + # via watchdog psutil==5.7.0 \ --hash=sha256:1413f4158eb50e110777c4f15d7c759521703bd6beb58926f1d562da40180058 \ --hash=sha256:298af2f14b635c3c7118fd9183843f4e73e681bb6f01e12284d4d70d48a60953 \ @@ -221,6 +224,9 @@ urllib3==1.25.8 \ --hash=sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc \ --hash=sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc \ # via -r requirements.in, requests +watchdog==0.10.2 \ + --hash=sha256:c560efb643faed5ef28784b2245cf8874f939569717a4a12826a173ac644456b \ + # via -r requirements.in werkzeug==0.15.5 \ --hash=sha256:87ae4e5b5366da2347eb3116c0e6c681a0e939a33b2805e2c0cbd282664932c4 \ --hash=sha256:a13b74dd3c45f758d4ebdb224be8f1ab8ef58b3c0ffc1783a8c7d9f4f50227e6 \ diff --git a/scripts/disable-dev-config.py b/scripts/disable-dev-config.py index 57fa3a26..cd471377 100755 --- a/scripts/disable-dev-config.py +++ b/scripts/disable-dev-config.py @@ -23,7 +23,7 @@ conf['general']['max_block_age'] = 2678400 conf['log']['file']['remove_on_exit'] = True conf['transports']['lan'] = True conf['transports']['tor'] = True -conf['transports']['manual_disk'] = True +conf['transports']['sneakernet'] = True json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4) diff --git a/src/etc/onionrvalues.py b/src/etc/onionrvalues.py index 192553ac..05825205 100755 --- a/src/etc/onionrvalues.py +++ b/src/etc/onionrvalues.py @@ -47,7 +47,7 @@ WSGI_SERVER_REQUEST_TIMEOUT_SECS = 120 MAX_NEW_PEER_QUEUE = 1000 -BLOCK_EXPORT_FILE_EXT = '.dat' +BLOCK_EXPORT_FILE_EXT = '.onionr' # Begin OnionrValues migrated values diff --git a/src/onionrcommands/daemonlaunch/__init__.py b/src/onionrcommands/daemonlaunch/__init__.py index d6d0733d..6b1374b1 100755 --- a/src/onionrcommands/daemonlaunch/__init__.py +++ b/src/onionrcommands/daemonlaunch/__init__.py @@ -37,6 +37,7 @@ from .killdaemon import kill_daemon # noqa from utils.boxprint import bordered from lan import LANManager from lan.server import LANServer +from sneakernet import sneakernet_import_thread """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -169,6 +170,8 @@ def daemon(): Thread(target=LANServer(shared_state).start_server, daemon=True).start() LANManager(shared_state).start() + if config.get('transports.sneakernet', True): + Thread(target=sneakernet_import_thread, daemon=True).start() communicator.startCommunicator(shared_state) clean_ephemeral_services() diff --git a/src/onionrcommands/exportblocks.py b/src/onionrcommands/exportblocks.py index df499fe2..404a6f92 100755 --- a/src/onionrcommands/exportblocks.py +++ b/src/onionrcommands/exportblocks.py @@ -8,6 +8,7 @@ import logger import onionrstorage from utils import createdirs from onionrutils import stringvalidators +from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT import filepaths import os @@ -31,23 +32,26 @@ from coredb import blockmetadb def _do_export(b_hash): createdirs.create_dirs() data = onionrstorage.getData(b_hash) - with open('%s/%s.dat' % (filepaths.export_location, - b_hash), 'wb') as export: + with open('%s/%s%s' % (filepaths.export_location, + b_hash, BLOCK_EXPORT_FILE_EXT), 'wb') as export: export.write(data) logger.info('Block exported as file', terminal=True) -def export_block(): +def export_block(*args): """Export block based on hash from stdin or argv.""" - try: - if not stringvalidators.validate_hash(sys.argv[2]): - raise ValueError - except (IndexError, ValueError): - logger.error('No valid block hash specified.', terminal=True) - sys.exit(1) + if args: + b_hash = args[0] else: - b_hash = sys.argv[2] - _do_export(b_hash) + try: + if not stringvalidators.validate_hash(sys.argv[2]): + raise ValueError + except (IndexError, ValueError): + logger.error('No valid block hash specified.', terminal=True) + sys.exit(1) + else: + b_hash = sys.argv[2] + _do_export(b_hash) export_block.onionr_help = ": Export block to " # type: ignore diff --git a/src/onionrutils/importnewblocks.py b/src/onionrutils/importnewblocks.py index 4ba614fb..bd2fa223 100644 --- a/src/onionrutils/importnewblocks.py +++ b/src/onionrutils/importnewblocks.py @@ -23,6 +23,7 @@ from onionrutils import blockmetadata from coredb import blockmetadb import filepaths import onionrcrypto as crypto +from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT def import_new_blocks(scanDir=''): ''' This function is intended to scan for new blocks ON THE DISK and import them @@ -33,19 +34,19 @@ def import_new_blocks(scanDir=''): scanDir = filepaths.block_data_location if not scanDir.endswith('/'): scanDir += '/' - for block in glob.glob(scanDir + "*.dat"): - if block.replace(scanDir, '').replace('.dat', '') not in blockList: + for block in glob.glob(scanDir + "*%s" % (BLOCK_EXPORT_FILE_EXT,)): + if block.replace(scanDir, '').replace(BLOCK_EXPORT_FILE_EXT, '') not in blockList: exist = True logger.info('Found new block on dist %s' % block, terminal=True) with open(block, 'rb') as newBlock: - block = block.replace(scanDir, '').replace('.dat', '') - if crypto.hashers.sha3_hash(newBlock.read()) == block.replace('.dat', ''): - blockmetadb.add_to_block_DB(block.replace('.dat', ''), dataSaved=True) - logger.info('Imported block %s.' % block, terminal=True) + block = block.replace(scanDir, '').replace(BLOCK_EXPORT_FILE_EXT, '') + if crypto.hashers.sha3_hash(newBlock.read()) == block.replace(BLOCK_EXPORT_FILE_EXT, ''): + blockmetadb.add_to_block_DB(block.replace(BLOCK_EXPORT_FILE_EXT, ''), dataSaved=True) + logger.info('Imported block %s' % block, terminal=True) blockmetadata.process_block_metadata(block) else: logger.warn('Failed to verify hash for %s' % block, terminal=True) if not exist: logger.info('No blocks found to import', terminal=True) -import_new_blocks.onionr_help = f"Scans the Onionr data directory under {filepaths.block_data_location} for new block files (.dat, .db not supported) to import" +import_new_blocks.onionr_help = f"Scans the Onionr data directory under {filepaths.block_data_location} for new block files (.db not supported) to import" diff --git a/src/runtests/__init__.py b/src/runtests/__init__.py index d8bac0bb..c9a6e3dc 100644 --- a/src/runtests/__init__.py +++ b/src/runtests/__init__.py @@ -15,6 +15,7 @@ from .osver import test_os_ver_endpoint from .clearnettor import test_clearnet_tor_request from .housekeeping import test_inserted_housekeeping from .lanservertest import test_lan_server +from .sneakernettest import test_sneakernet_import """ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -39,7 +40,8 @@ RUN_TESTS = [uicheck.check_ui, test_os_ver_endpoint, test_clearnet_tor_request, test_inserted_housekeeping, - test_lan_server + test_lan_server, + sneakernettest.test_sneakernet_import ] SUCCESS_FILE = os.path.dirname(os.path.realpath(__file__)) + '/../../tests/runtime-result.txt' diff --git a/src/runtests/sneakernettest.py b/src/runtests/sneakernettest.py new file mode 100644 index 00000000..ac48ad30 --- /dev/null +++ b/src/runtests/sneakernettest.py @@ -0,0 +1,27 @@ +import os +from shutil import move + +from onionrblocks import insert +from onionrstorage import deleteBlock +from onionrcommands.exportblocks import export_block +from filepaths import export_location, block_data_location, data_nonce_file +from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT +from onionrstorage.removeblock import remove_block +from onionrstorage import deleteBlock +from coredb.blockmetadb import get_block_list +from utils import bettersleep +from gevent import sleep + +def test_sneakernet_import(test_manager): + in_db = lambda b: b in get_block_list() + bl = insert(os.urandom(10)) + assert in_db(bl) + export_block(bl) + assert os.path.exists(export_location + bl + BLOCK_EXPORT_FILE_EXT) + remove_block(bl) + deleteBlock(bl) + assert not in_db(bl) + os.remove(data_nonce_file) + move(export_location + bl + BLOCK_EXPORT_FILE_EXT, block_data_location) + sleep(1) + assert in_db(bl) diff --git a/src/sneakernet/__init__.py b/src/sneakernet/__init__.py new file mode 100644 index 00000000..6535615c --- /dev/null +++ b/src/sneakernet/__init__.py @@ -0,0 +1,54 @@ +"""Onionr - Private P2P Communication. + +Detect new block files in a given directory +""" +import os + +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + +import config +from filepaths import block_data_location +from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT +from onionrblocks.blockimporter import import_block_from_data +""" + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + +watch_paths = config.get('transports.sneakernet.paths', list([])) +if block_data_location not in watch_paths: + watch_paths.append(block_data_location) + + +class _Importer(FileSystemEventHandler): + @staticmethod + def on_created(event): + if not event.src_path.endswith(BLOCK_EXPORT_FILE_EXT): + return + with open(event.src_path, 'rb') as block_file: + import_block_from_data(block_file.read()) + if block_data_location in event.src_path: + try: + os.remove(event.src_path) + except FileNotFoundError: + pass + + +def sneakernet_import_thread(): + observer = Observer() + for path in watch_paths: + observer.schedule(_Importer(), path, recursive=True) + observer.start() + while observer.isAlive(): + observer.join(60) \ No newline at end of file diff --git a/static-data/default-plugins/metadataprocessor/info.json b/static-data/default-plugins/metadataprocessor/info.json deleted file mode 100755 index 355d98f1..00000000 --- a/static-data/default-plugins/metadataprocessor/info.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name" : "metadataprocessor", - "version" : "1.0", - "author" : "onionr" -} diff --git a/static-data/default-plugins/metadataprocessor/main.py b/static-data/default-plugins/metadataprocessor/main.py deleted file mode 100755 index 5ee1fd18..00000000 --- a/static-data/default-plugins/metadataprocessor/main.py +++ /dev/null @@ -1,59 +0,0 @@ -''' - Onionr - Private P2P Communication - - This processes metadata for Onionr blocks -''' -''' - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' - -# useful libraries -import logger, config -import os, sys, json, time, random, shutil, base64, getpass, datetime, re -import onionrusers, onionrexceptions -from onionrutils import stringvalidators - -plugin_name = 'metadataprocessor' - -# event listeners - -def _processForwardKey(api, myBlock): - ''' - Get the forward secrecy key specified by the user for us to use - ''' - peer = onionrusers.OnionrUser(myBlock.signer) - key = myBlock.getMetadata('newFSKey') - - # We don't need to validate here probably, but it helps - if stringvalidators.validate_pub_key(key): - peer.addForwardKey(key) - else: - raise onionrexceptions.InvalidPubkey("%s is not a valid pubkey key" % (key,)) - -def on_processblocks(api, data=None): - # Generally fired by utils. - myBlock = api.data['block'] - blockType = api.data['type'] - # Process specific block types - - # forwardKey blocks, add a new forward secrecy key for a peer - if blockType == 'forwardKey': - if api.data['validSig'] == True: - _processForwardKey(api, myBlock) - -def on_init(api, data = None): - - pluginapi = api - - return diff --git a/static-data/default_config.json b/static-data/default_config.json index 453b69e1..e7b38ea9 100755 --- a/static-data/default_config.json +++ b/static-data/default_config.json @@ -63,7 +63,7 @@ }, "transports": { "lan": true, - "manual_disk": true, + "sneakernet": true, "tor": true }, "ui": { diff --git a/tests/runtime-result.txt b/tests/runtime-result.txt index 0a2bb8a5..52b64ce6 100644 --- a/tests/runtime-result.txt +++ b/tests/runtime-result.txt @@ -1 +1 @@ -1585525008 \ No newline at end of file +1585619396 \ No newline at end of file diff --git a/tests/test_default_config_json.py b/tests/test_default_config_json.py index 4905c1f7..ae5ab5a8 100644 --- a/tests/test_default_config_json.py +++ b/tests/test_default_config_json.py @@ -58,7 +58,7 @@ class OnionrConfig(unittest.TestCase): self.assertEqual(conf['tor']['use_bridge'], False) self.assertEqual(conf['tor']['use_existing_tor'], False) self.assertEqual(conf['transports']['lan'], True) - self.assertEqual(conf['transports']['manual_disk'], True) + self.assertEqual(conf['transports']['sneakernet'], True) self.assertEqual(conf['transports']['tor'], True) self.assertEqual(conf['ui']['theme'], 'dark') unittest.main() diff --git a/tests/test_onionrvalues.py b/tests/test_onionrvalues.py index c1ae1430..c7de3848 100644 --- a/tests/test_onionrvalues.py +++ b/tests/test_onionrvalues.py @@ -19,6 +19,6 @@ class TestOnionrValues(unittest.TestCase): self.assertEqual(onionrvalues.MAX_BLOCK_CLOCK_SKEW, 120) def test_block_export_ext(self): - self.assertEqual(onionrvalues.BLOCK_EXPORT_FILE_EXT, '.dat') + self.assertEqual(onionrvalues.BLOCK_EXPORT_FILE_EXT, '.onionr') unittest.main()