dxs
/
onionr
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added sneakernet auto importing 

exportblocks now takes argument
master
Kevin Froman 2020-03-30 03:23:59 -05:00
parent 160469f50f
commit 1bd0aa9419
17 changed files with 131 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.vscode/tasks.json vendored
View File

@ -46,6 +46,13 @@
"command": "scripts/disable-dev-config.py",
"group": "dev",
"problemMatcher": []
},
{
"label": "Format default config",
"type": "process",
"command": "scripts/pretty-default-config.py",
"group": "dev",
"problemMatcher": []
}
]
}

2
requirements.in
View File

@ -8,9 +8,9 @@ stem==1.8.0
deadsimplekv==0.3.1
unpaddedbase32==0.2.0
streamedrequests==1.0.0
jinja2==2.11.1
toomanyobjs==1.1.0
niceware==0.2.1
psutil==5.7.0
filenuke==0.0.0
mimcvdf==1.0.0
watchdog==0.10.2

8
requirements.txt
View File

@ -115,7 +115,7 @@ itsdangerous==1.1.0 \
jinja2==2.11.1 \
--hash=sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250 \
--hash=sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49 \
# via -r requirements.in, flask
# via flask
markupsafe==1.1.1 \
--hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \
--hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \
@ -154,6 +154,9 @@ niceware==0.2.1 \
--hash=sha256:0f8b192f2a1e800e068474f6e208be9c7e2857664b33a96f4045340de4e5c69c \
--hash=sha256:cf2dc0e1567d36d067c61b32fed0f1b9c4534ed511f9eeead4ba548d03b5c9eb \
# via -r requirements.in
pathtools==0.1.2 \
--hash=sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0 \
# via watchdog
psutil==5.7.0 \
--hash=sha256:1413f4158eb50e110777c4f15d7c759521703bd6beb58926f1d562da40180058 \
--hash=sha256:298af2f14b635c3c7118fd9183843f4e73e681bb6f01e12284d4d70d48a60953 \
@ -221,6 +224,9 @@ urllib3==1.25.8 \
--hash=sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc \
--hash=sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc \
# via -r requirements.in, requests
watchdog==0.10.2 \
--hash=sha256:c560efb643faed5ef28784b2245cf8874f939569717a4a12826a173ac644456b \
# via -r requirements.in
werkzeug==0.15.5 \
--hash=sha256:87ae4e5b5366da2347eb3116c0e6c681a0e939a33b2805e2c0cbd282664932c4 \
--hash=sha256:a13b74dd3c45f758d4ebdb224be8f1ab8ef58b3c0ffc1783a8c7d9f4f50227e6 \

2
scripts/disable-dev-config.py
View File

@ -23,7 +23,7 @@ conf['general']['max_block_age'] = 2678400
conf['log']['file']['remove_on_exit'] = True
conf['transports']['lan'] = True
conf['transports']['tor'] = True
conf['transports']['manual_disk'] = True
conf['transports']['sneakernet'] = True
json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)

2
src/etc/onionrvalues.py
View File

@ -47,7 +47,7 @@ WSGI_SERVER_REQUEST_TIMEOUT_SECS = 120
MAX_NEW_PEER_QUEUE = 1000
BLOCK_EXPORT_FILE_EXT = '.dat'
BLOCK_EXPORT_FILE_EXT = '.onionr'
# Begin OnionrValues migrated values

3
src/onionrcommands/daemonlaunch/__init__.py
View File

@ -37,6 +37,7 @@ from .killdaemon import kill_daemon # noqa
from utils.boxprint import bordered
from lan import LANManager
from lan.server import LANServer
from sneakernet import sneakernet_import_thread
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -169,6 +170,8 @@ def daemon():
Thread(target=LANServer(shared_state).start_server,
daemon=True).start()
LANManager(shared_state).start()
if config.get('transports.sneakernet', True):
Thread(target=sneakernet_import_thread, daemon=True).start()
communicator.startCommunicator(shared_state)
clean_ephemeral_services()

10
src/onionrcommands/exportblocks.py
View File

@ -8,6 +8,7 @@ import logger
import onionrstorage
from utils import createdirs
from onionrutils import stringvalidators
from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
import filepaths
import os
@ -31,14 +32,17 @@ from coredb import blockmetadb
def _do_export(b_hash):
createdirs.create_dirs()
data = onionrstorage.getData(b_hash)
with open('%s/%s.dat' % (filepaths.export_location,
b_hash), 'wb') as export:
with open('%s/%s%s' % (filepaths.export_location,
b_hash, BLOCK_EXPORT_FILE_EXT), 'wb') as export:
export.write(data)
logger.info('Block exported as file', terminal=True)
def export_block():
def export_block(*args):
"""Export block based on hash from stdin or argv."""
if args:
b_hash = args[0]
else:
try:
if not stringvalidators.validate_hash(sys.argv[2]):
raise ValueError

15
src/onionrutils/importnewblocks.py
View File

@ -23,6 +23,7 @@ from onionrutils import blockmetadata
from coredb import blockmetadb
import filepaths
import onionrcrypto as crypto
from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
def import_new_blocks(scanDir=''):
'''
This function is intended to scan for new blocks ON THE DISK and import them
@ -33,19 +34,19 @@ def import_new_blocks(scanDir=''):
scanDir = filepaths.block_data_location
if not scanDir.endswith('/'):
scanDir += '/'
for block in glob.glob(scanDir + "*.dat"):
if block.replace(scanDir, '').replace('.dat', '') not in blockList:
for block in glob.glob(scanDir + "*%s" % (BLOCK_EXPORT_FILE_EXT,)):
if block.replace(scanDir, '').replace(BLOCK_EXPORT_FILE_EXT, '') not in blockList:
exist = True
logger.info('Found new block on dist %s' % block, terminal=True)
with open(block, 'rb') as newBlock:
block = block.replace(scanDir, '').replace('.dat', '')
if crypto.hashers.sha3_hash(newBlock.read()) == block.replace('.dat', ''):
blockmetadb.add_to_block_DB(block.replace('.dat', ''), dataSaved=True)
logger.info('Imported block %s.' % block, terminal=True)
block = block.replace(scanDir, '').replace(BLOCK_EXPORT_FILE_EXT, '')
if crypto.hashers.sha3_hash(newBlock.read()) == block.replace(BLOCK_EXPORT_FILE_EXT, ''):
blockmetadb.add_to_block_DB(block.replace(BLOCK_EXPORT_FILE_EXT, ''), dataSaved=True)
logger.info('Imported block %s' % block, terminal=True)
blockmetadata.process_block_metadata(block)
else:
logger.warn('Failed to verify hash for %s' % block, terminal=True)
if not exist:
logger.info('No blocks found to import', terminal=True)
import_new_blocks.onionr_help = f"Scans the Onionr data directory under {filepaths.block_data_location} for new block files (.dat, .db not supported) to import"
import_new_blocks.onionr_help = f"Scans the Onionr data directory under {filepaths.block_data_location} for new block files (.db not supported) to import"

4
src/runtests/__init__.py
View File

@ -15,6 +15,7 @@ from .osver import test_os_ver_endpoint
from .clearnettor import test_clearnet_tor_request
from .housekeeping import test_inserted_housekeeping
from .lanservertest import test_lan_server
from .sneakernettest import test_sneakernet_import
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -39,7 +40,8 @@ RUN_TESTS = [uicheck.check_ui,
test_os_ver_endpoint,
test_clearnet_tor_request,
test_inserted_housekeeping,
test_lan_server
test_lan_server,
sneakernettest.test_sneakernet_import
]
SUCCESS_FILE = os.path.dirname(os.path.realpath(__file__)) + '/../../tests/runtime-result.txt'

27
src/runtests/sneakernettest.py Normal file
View File

@ -0,0 +1,27 @@
import os
from shutil import move
from onionrblocks import insert
from onionrstorage import deleteBlock
from onionrcommands.exportblocks import export_block
from filepaths import export_location, block_data_location, data_nonce_file
from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
from onionrstorage.removeblock import remove_block
from onionrstorage import deleteBlock
from coredb.blockmetadb import get_block_list
from utils import bettersleep
from gevent import sleep
def test_sneakernet_import(test_manager):
in_db = lambda b: b in get_block_list()
bl = insert(os.urandom(10))
assert in_db(bl)
export_block(bl)
assert os.path.exists(export_location + bl + BLOCK_EXPORT_FILE_EXT)
remove_block(bl)
deleteBlock(bl)
assert not in_db(bl)
os.remove(data_nonce_file)
move(export_location + bl + BLOCK_EXPORT_FILE_EXT, block_data_location)
sleep(1)
assert in_db(bl)

54
src/sneakernet/__init__.py Normal file
View File

@ -0,0 +1,54 @@
"""Onionr - Private P2P Communication.
Detect new block files in a given directory
"""
import os
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
import config
from filepaths import block_data_location
from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
from onionrblocks.blockimporter import import_block_from_data
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
watch_paths = config.get('transports.sneakernet.paths', list([]))
if block_data_location not in watch_paths:
watch_paths.append(block_data_location)
class _Importer(FileSystemEventHandler):
@staticmethod
def on_created(event):
if not event.src_path.endswith(BLOCK_EXPORT_FILE_EXT):
return
with open(event.src_path, 'rb') as block_file:
import_block_from_data(block_file.read())
if block_data_location in event.src_path:
try:
os.remove(event.src_path)
except FileNotFoundError:
pass
def sneakernet_import_thread():
observer = Observer()
for path in watch_paths:
observer.schedule(_Importer(), path, recursive=True)
observer.start()
while observer.isAlive():
observer.join(60)

5
static-data/default-plugins/metadataprocessor/info.json
View File

@ -1,5 +0,0 @@
{
"name" : "metadataprocessor",
"version" : "1.0",
"author" : "onionr"
}

59
static-data/default-plugins/metadataprocessor/main.py
View File

@ -1,59 +0,0 @@
'''
Onionr - Private P2P Communication
This processes metadata for Onionr blocks
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
# useful libraries
import logger, config
import os, sys, json, time, random, shutil, base64, getpass, datetime, re
import onionrusers, onionrexceptions
from onionrutils import stringvalidators
plugin_name = 'metadataprocessor'
# event listeners
def _processForwardKey(api, myBlock):
'''
Get the forward secrecy key specified by the user for us to use
'''
peer = onionrusers.OnionrUser(myBlock.signer)
key = myBlock.getMetadata('newFSKey')
# We don't need to validate here probably, but it helps
if stringvalidators.validate_pub_key(key):
peer.addForwardKey(key)
else:
raise onionrexceptions.InvalidPubkey("%s is not a valid pubkey key" % (key,))
def on_processblocks(api, data=None):
# Generally fired by utils.
myBlock = api.data['block']
blockType = api.data['type']
# Process specific block types
# forwardKey blocks, add a new forward secrecy key for a peer
if blockType == 'forwardKey':
if api.data['validSig'] == True:
_processForwardKey(api, myBlock)
def on_init(api, data = None):
pluginapi = api
return

2
static-data/default_config.json
View File

@ -63,7 +63,7 @@
},
"transports": {
"lan": true,
"manual_disk": true,
"sneakernet": true,
"tor": true
},
"ui": {

2
tests/runtime-result.txt
View File

@ -1 +1 @@
1585525008
1585619396

2
tests/test_default_config_json.py
View File

@ -58,7 +58,7 @@ class OnionrConfig(unittest.TestCase):
self.assertEqual(conf['tor']['use_bridge'], False)
self.assertEqual(conf['tor']['use_existing_tor'], False)
self.assertEqual(conf['transports']['lan'], True)
self.assertEqual(conf['transports']['manual_disk'], True)
self.assertEqual(conf['transports']['sneakernet'], True)
self.assertEqual(conf['transports']['tor'], True)
self.assertEqual(conf['ui']['theme'], 'dark')
unittest.main()

2
tests/test_onionrvalues.py
View File

@ -19,6 +19,6 @@ class TestOnionrValues(unittest.TestCase):
self.assertEqual(onionrvalues.MAX_BLOCK_CLOCK_SKEW, 120)
def test_block_export_ext(self):
self.assertEqual(onionrvalues.BLOCK_EXPORT_FILE_EXT, '.dat')
self.assertEqual(onionrvalues.BLOCK_EXPORT_FILE_EXT, '.onionr')
unittest.main()