added sneakernet auto importing

exportblocks now takes argument

src/etc/onionrvalues.py

@@ -47,7 +47,7 @@ WSGI_SERVER_REQUEST_TIMEOUT_SECS = 120
 
 MAX_NEW_PEER_QUEUE = 1000
 
-BLOCK_EXPORT_FILE_EXT = '.dat'
+BLOCK_EXPORT_FILE_EXT = '.onionr'
 
 # Begin OnionrValues migrated values
 

src/onionrcommands/daemonlaunch/__init__.py

@@ -37,6 +37,7 @@ from .killdaemon import kill_daemon  # noqa
 from utils.boxprint import bordered
 from lan import LANManager
 from lan.server import LANServer
+from sneakernet import sneakernet_import_thread
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -169,6 +170,8 @@ def daemon():
         Thread(target=LANServer(shared_state).start_server,
                daemon=True).start()
         LANManager(shared_state).start()
+    if config.get('transports.sneakernet', True):
+        Thread(target=sneakernet_import_thread, daemon=True).start()
     communicator.startCommunicator(shared_state)
 
     clean_ephemeral_services()

src/onionrcommands/exportblocks.py

@@ -8,6 +8,7 @@ import logger
 import onionrstorage
 from utils import createdirs
 from onionrutils import stringvalidators
+from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
 import filepaths
 
 import os
@@ -31,23 +32,26 @@ from coredb import blockmetadb
 def _do_export(b_hash):
     createdirs.create_dirs()
     data = onionrstorage.getData(b_hash)
-    with open('%s/%s.dat' % (filepaths.export_location,
-                             b_hash), 'wb') as export:
+    with open('%s/%s%s' % (filepaths.export_location,
+                             b_hash, BLOCK_EXPORT_FILE_EXT), 'wb') as export:
         export.write(data)
         logger.info('Block exported as file', terminal=True)
 
 
-def export_block():
+def export_block(*args):
     """Export block based on hash from stdin or argv."""
-    try:
-        if not stringvalidators.validate_hash(sys.argv[2]):
-            raise ValueError
-    except (IndexError, ValueError):
-        logger.error('No valid block hash specified.', terminal=True)
-        sys.exit(1)
+    if args:
+        b_hash = args[0]
     else:
-        b_hash = sys.argv[2]
-        _do_export(b_hash)
+        try:
+            if not stringvalidators.validate_hash(sys.argv[2]):
+                raise ValueError
+        except (IndexError, ValueError):
+            logger.error('No valid block hash specified.', terminal=True)
+            sys.exit(1)
+        else:
+            b_hash = sys.argv[2]
+    _do_export(b_hash)
 
 
 export_block.onionr_help = "<block hash>: Export block to "  # type: ignore

src/onionrutils/importnewblocks.py

@@ -23,6 +23,7 @@ from onionrutils import blockmetadata
 from coredb import blockmetadb
 import filepaths
 import onionrcrypto as crypto
+from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
 def import_new_blocks(scanDir=''):
     '''
         This function is intended to scan for new blocks ON THE DISK and import them
@@ -33,19 +34,19 @@ def import_new_blocks(scanDir=''):
         scanDir = filepaths.block_data_location
     if not scanDir.endswith('/'):
         scanDir += '/'
-    for block in glob.glob(scanDir + "*.dat"):
-        if block.replace(scanDir, '').replace('.dat', '') not in blockList:
+    for block in glob.glob(scanDir + "*%s" % (BLOCK_EXPORT_FILE_EXT,)):
+        if block.replace(scanDir, '').replace(BLOCK_EXPORT_FILE_EXT, '') not in blockList:
             exist = True
             logger.info('Found new block on dist %s' % block, terminal=True)
             with open(block, 'rb') as newBlock:
-                block = block.replace(scanDir, '').replace('.dat', '')
-                if crypto.hashers.sha3_hash(newBlock.read()) == block.replace('.dat', ''):
-                    blockmetadb.add_to_block_DB(block.replace('.dat', ''), dataSaved=True)
-                    logger.info('Imported block %s.' % block, terminal=True)
+                block = block.replace(scanDir, '').replace(BLOCK_EXPORT_FILE_EXT, '')
+                if crypto.hashers.sha3_hash(newBlock.read()) == block.replace(BLOCK_EXPORT_FILE_EXT, ''):
+                    blockmetadb.add_to_block_DB(block.replace(BLOCK_EXPORT_FILE_EXT, ''), dataSaved=True)
+                    logger.info('Imported block %s' % block, terminal=True)
                     blockmetadata.process_block_metadata(block)
                 else:
                     logger.warn('Failed to verify hash for %s' % block, terminal=True)
     if not exist:
         logger.info('No blocks found to import', terminal=True)
 
-import_new_blocks.onionr_help = f"Scans the Onionr data directory under {filepaths.block_data_location} for new block files (.dat, .db not supported) to import"
+import_new_blocks.onionr_help = f"Scans the Onionr data directory under {filepaths.block_data_location} for new block files (.db not supported) to import"

src/runtests/__init__.py

@@ -15,6 +15,7 @@ from .osver import test_os_ver_endpoint
 from .clearnettor import test_clearnet_tor_request
 from .housekeeping import test_inserted_housekeeping
 from .lanservertest import test_lan_server
+from .sneakernettest import test_sneakernet_import
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -39,7 +40,8 @@ RUN_TESTS = [uicheck.check_ui,
              test_os_ver_endpoint,
              test_clearnet_tor_request,
              test_inserted_housekeeping,
-             test_lan_server
+             test_lan_server,
+             sneakernettest.test_sneakernet_import
              ]
 
 SUCCESS_FILE = os.path.dirname(os.path.realpath(__file__)) + '/../../tests/runtime-result.txt'

src/runtests/sneakernettest.py

@@ -0,0 +1,27 @@
+import os
+from shutil import move
+
+from onionrblocks import insert
+from onionrstorage import deleteBlock
+from onionrcommands.exportblocks import export_block
+from filepaths import export_location, block_data_location, data_nonce_file
+from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
+from onionrstorage.removeblock import remove_block
+from onionrstorage import deleteBlock
+from coredb.blockmetadb import get_block_list
+from utils import bettersleep
+from gevent import sleep
+
+def test_sneakernet_import(test_manager):
+    in_db = lambda b: b in get_block_list()
+    bl = insert(os.urandom(10))
+    assert in_db(bl)
+    export_block(bl)
+    assert os.path.exists(export_location + bl + BLOCK_EXPORT_FILE_EXT)
+    remove_block(bl)
+    deleteBlock(bl)
+    assert not in_db(bl)
+    os.remove(data_nonce_file)
+    move(export_location + bl + BLOCK_EXPORT_FILE_EXT, block_data_location)
+    sleep(1)
+    assert in_db(bl)

src/sneakernet/__init__.py

@@ -0,0 +1,54 @@
+"""Onionr - Private P2P Communication.
+
+Detect new block files in a given directory
+"""
+import os
+
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+import config
+from filepaths import block_data_location
+from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
+from onionrblocks.blockimporter import import_block_from_data
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+watch_paths = config.get('transports.sneakernet.paths', list([]))
+if block_data_location not in watch_paths:
+    watch_paths.append(block_data_location)
+
+
+class _Importer(FileSystemEventHandler):
+    @staticmethod
+    def on_created(event):
+        if not event.src_path.endswith(BLOCK_EXPORT_FILE_EXT):
+            return
+        with open(event.src_path, 'rb') as block_file:
+            import_block_from_data(block_file.read())
+            if block_data_location in event.src_path:
+                try:
+                    os.remove(event.src_path)
+                except FileNotFoundError:
+                    pass
+
+
+def sneakernet_import_thread():
+    observer = Observer()
+    for path in watch_paths:
+        observer.schedule(_Importer(), path, recursive=True)
+    observer.start()
+    while observer.isAlive():
+        observer.join(60)