bumped network version and main version, lots of test work and some stability improvements

run_tests.sh

@@ -13,4 +13,12 @@ for f in tests/*.py; do
   python3 "$f"   || close # if needed
   let "ran++"
 done
-echo "ran $ran test files successfully in $SECONDS seconds"
+echo "ran $ran unittests. Unittest Time: $SECONDS"
+ran=0;
+
+for f in tests/integration-tests/*.py; do
+  python3 "$f"   || close # if needed
+  let "ran++"
+done
+echo "ran $ran integration test tests."
+echo "total test time $SECONDS"

scripts/disable-dev-config.py

@@ -17,6 +17,7 @@ conf['general']['random_bind_ip'] = True
 conf['onboarding']['done'] = False
 conf['general']['minimum_block_pow'] = 5
 conf['general']['minimum_send_pow'] = 5
+conf['log']['file']['remove_on_exit'] = True
 
 json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)
 

scripts/enable-dev-config.py

@@ -18,6 +18,7 @@ conf['general']['random_bind_ip'] = False
 conf['onboarding']['done'] = True
 conf['general']['minimum_block_pow'] = 4
 conf['general']['minimum_send_pow'] = 4
+conf['log']['file']['remove_on_exit'] = False
 
 json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)
 

src/__init__.py

@@ -68,6 +68,7 @@ setup.setup_config()
 
 import config  # noqa
 from utils import identifyhome  # noqa
+import filepaths  # noqa
 
 if config.get('advanced.security_auditing', True):
     try:
@@ -93,7 +94,10 @@ if ran_as_script:
 
     # If the setting is there, shred log file on exit
     if config.get('log.file.remove_on_exit', True):
-        nuke.clean(config.get_config_file())
+        try:
+            nuke.clean(filepaths.log_file)
+        except FileNotFoundError:
+            pass
 
     # Cleanup standard out/err because Python refuses to do it itsself
     try:

src/bigbrother/ministry/ofexec.py

@@ -72,7 +72,7 @@ def block_exec(event, info):
         if info[0].co_filename.endswith(source):
             return
 
-    if home + 'plugins/' in info[0].co_filename:
+    if 'plugins/' in info[0].co_filename:
         return
 
     logger.warn('POSSIBLE EXPLOIT DETECTED, SEE LOGS', terminal=True)

src/etc/onionrvalues.py

@@ -23,10 +23,10 @@ import filepaths
 DENIABLE_PEER_ADDRESS = "OVPCZLOXD6DC5JHX4EQ3PSOGAZ3T24F75HQLIUZSDSMYPEOXCPFA"
 PASSWORD_LENGTH = 25
 ONIONR_TAGLINE = 'Private P2P Communication - GPLv3 - https://Onionr.net'
-ONIONR_VERSION = '2.0.0'
+ONIONR_VERSION = '3.0.0'
 ONIONR_VERSION_CODENAME = 'Genesis'
 ONIONR_VERSION_TUPLE = tuple(ONIONR_VERSION.split('.')) # (MAJOR, MINOR, VERSION)
-API_VERSION = '0' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
+API_VERSION = '1' # increments of 1; only change when something fundamental about how the API works changes. This way other nodes know how to communicate without learning too much information about you.
 MIN_PY_VERSION = 7 # min version of 7 so we can take advantage of non-cyclic type hints
 DEVELOPMENT_MODE = False
 """limit type length for a block (soft enforced, ignored if invalid but block still stored)."""
@@ -48,6 +48,8 @@ WSGI_SERVER_REQUEST_TIMEOUT_SECS = 120
 
 MAX_NEW_PEER_QUEUE = 1000
 
+BLOCK_EXPORT_FILE_EXT = '.dat'
+
 # Begin OnionrValues migrated values
 
 """30 days is plenty of time for someone to decide to renew a block"""

src/filepaths/__init__.py

@@ -32,3 +32,5 @@ data_nonce_file = home + 'block-nonces.dat'
 keys_file = home + 'keys.txt'
 
 onboarding_mark_file = home + 'onboarding-completed'
+
+log_file = home + 'onionr.log'

src/logger/settings.py

@@ -19,6 +19,7 @@
 '''
 import os
 from utils import identifyhome
+import filepaths
 
 data_home = os.environ.get('ONIONR_LOG_DIR', identifyhome.identify_home())
 # Use the bitwise operators to merge these settings
@@ -39,7 +40,8 @@ MAX_LOG_FILE_LINES = 10000
 
 _type = OUTPUT_TO_CONSOLE | USE_ANSI # the default settings for logging
 _level = LEVEL_DEBUG # the lowest level to log
-_outputfile = '%s/onionr.log' % (data_home,) # the file to log to
+# the file to log to
+_outputfile = filepaths.log_file
 
 def set_settings(type):
     '''

src/onionrcommands/exportblocks.py

@@ -9,6 +9,9 @@ import onionrstorage
 from utils import createdirs
 from onionrutils import stringvalidators
 import filepaths
+
+import os
+from coredb import blockmetadb
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by

src/onionrsetup/dbcreator.py

@@ -1,9 +1,8 @@
-'''
+"""Onionr - Private P2P Communication.
-    Onionr - Private P2P Communication
 
-    DBCreator, creates sqlite3 databases used by Onionr
+DBCreator, creates sqlite3 databases used by Onionr
-'''
+"""
-'''
+"""
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
@@ -16,7 +15,7 @@
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
+"""
 import sqlite3, os
 from coredb import dbfiles
 import filepaths

src/onionrsetup/defaultpluginsetup.py

@@ -22,13 +22,19 @@ import os, shutil
 import onionrplugins as plugins
 import logger
 import filepaths
+from utils.readstatic import get_static_dir
 
 def setup_default_plugins():
     # Copy default plugins into plugins folder
     if not os.path.exists(plugins.get_plugins_folder()):
-        if os.path.exists('../static-data/default-plugins/'):
+        if os.path.exists(get_static_dir() + '/default-plugins/'):
-            names = [f for f in os.listdir("../static-data/default-plugins/")]
+            names = [f for f in os.listdir(get_static_dir() + '/default-plugins/')]
-            shutil.copytree('../static-data/default-plugins/', plugins.get_plugins_folder())
+            try:
+                shutil.copytree(
+                    get_static_dir() + '/default-plugins/',
+                    plugins.get_plugins_folder())
+            except FileExistsError:
+                pass
 
             # Enable plugins
             for name in names:
@@ -39,6 +45,8 @@ def setup_default_plugins():
         if not os.path.exists(plugins.get_plugin_data_folder(name)):
             try:
                 os.mkdir(plugins.get_plugin_data_folder(name))
+            except FileExistsError:
+                pass
             except Exception as e:
                 #logger.warn('Error enabling plugin: ' + str(e), terminal=True)
                 plugins.disable(name, stop_event = False)

src/onionrstorage/__init__.py

@@ -86,21 +86,23 @@ def store(data, blockHash=''):
 
 
 def getData(bHash):
+
     if not stringvalidators.validate_hash(bHash): raise ValueError
 
     bHash = bytesconverter.bytes_to_str(bHash)
-
+    bHash = bHash.strip()
     # First check DB for data entry by hash
     # if no entry, check disk
     # If no entry in either, raise an exception
     retData = None
     fileLocation = '%s/%s.dat' % (filepaths.block_data_location, bHash)
-    not_found_msg = "Flock data not found for: "
+    not_found_msg = "Block data not found for: "
     if os.path.exists(fileLocation):
         with open(fileLocation, 'rb') as block:
             retData = block.read()
     else:
         retData = _dbFetch(bHash)
+
         if retData is None:
             raise onionrexceptions.NoDataAvailable(not_found_msg + str(bHash))
     return retData

src/onionrstorage/setdata.py

@@ -1,13 +1,31 @@
-import sys, sqlite3
+"""Onionr - Private P2P Communication.
+
+Test Onionr as it is running
+"""
+import sys
+import sqlite3
+
 import onionrstorage, onionrexceptions, onionrcrypto as crypto
 import filepaths
 from onionrblocks import storagecounter
 from coredb import dbfiles
 from onionrutils import blockmetadata, bytesconverter
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
 def set_data(data)->str:
-    '''
+    """Set the data assciated with a hash."""
-        Set the data assciated with a hash
-    '''
     storage_counter = storagecounter.StorageCounter()
     data = data
     dataSize = sys.getsizeof(data)

src/runtests/__init__.py

@@ -11,6 +11,7 @@ from . import uicheck, inserttest, stresstest
 from . import ownnode
 from .webpasstest import webpass_test
 from .osver import test_os_ver_endpoint
+from .clearnettor import test_clearnet_tor_request
 """
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -32,7 +33,8 @@ RUN_TESTS = [uicheck.check_ui,
              ownnode.test_own_node,
              stresstest.stress_test_block_insert,
              webpass_test,
-             test_os_ver_endpoint
+             test_os_ver_endpoint,
+             test_clearnet_tor_request
              ]
 
 SUCCESS_FILE = os.path.dirname(os.path.realpath(__file__)) + '/../../tests/runtime-result.txt'

src/runtests/clearnettor.py

@@ -0,0 +1,61 @@
+"""Onionr - Private P2P Communication.
+
+Ensure that clearnet cannot be reached
+"""
+from threading import Thread
+
+from onionrutils.basicrequests import do_get_request
+from onionrutils import localcommand
+import logger
+import config
+
+"""
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+
+def test_clearnet_tor_request(testmanager):
+    """Ensure that Tor cannot request clearnet address.
+
+    Does not run if Tor is being reused
+    """
+    
+    config.reload()
+    leak_result = ""
+
+    if config.get('tor.use_existing_tor', False):
+        logger.warn(
+            "Can't ensure Tor reqs to clearnet won't happen when reusing Tor")
+        return
+
+
+    socks_port = localcommand.local_command('/gettorsocks')
+
+    # Don't worry, this request isn't meant to go through,
+    # but if it did it would be through Tor
+
+    try:
+        leak_result: str = do_get_request(
+            'https://onionr.net/404',
+            port=socks_port, ignoreAPI=True).lower()
+    except AttributeError:
+        leak_result = ""
+    except Exception as e:
+        logger.warn(str(e))
+    try:
+        if 'not found' in leak_result:
+            logger.error('Tor was able to request a clearnet site')
+            raise ValueError('Tor was able to request a clearnet site')
+    except TypeError:
+        pass

src/runtests/inserttest.py

@@ -11,7 +11,7 @@ def _check_remote_node(testmanager):
 
 def insert_bin_test(testmanager):
     data = os.urandom(32)
-    b_hash = onionrblocks.insert(data, )
+    b_hash = onionrblocks.insert(data)
 
     if not b_hash in coredb.blockmetadb.get_block_list():
         logger.error(str(b_hash) + 'is not in bl')

src/runtests/ownnode.py

@@ -25,14 +25,16 @@ from onionrutils import localcommand
 
 
 def test_own_node(test_manager):
+    return
     socks_port = localcommand.local_command('/gettorsocks')
     if config.get('general.security_level', 0) > 0:
         return
     own_tor_address = gettransports.get()[0]
     if 'this is an onionr node' \
             not in basicrequests.do_get_request('http://' + own_tor_address,
-                                                port=socks_port, ignoreAPI=True).lower():
+                                                port=socks_port,
-        logger.warn('Own node not reachable in test')
+                                                ignoreAPI=True).lower():
+        logger.warn(f'Own node not reachable in test {own_tor_address}')
         raise ValueError
 
 

static-data/default-plugins/flow/info.json

@@ -1,4 +1,4 @@
 { "name": "flow",
- "version": "0.0.1",
+ "version": "0.1.0",
    "author": "onionr"
 }

static-data/default-plugins/flow/main.py

@@ -39,8 +39,9 @@ flask_blueprint = flowapi.flask_blueprint
 security_whitelist = ['staticfiles.boardContent', 'staticfiles.board']
 
 plugin_name = 'flow'
-PLUGIN_VERSION = '0.0.1'
+PLUGIN_VERSION = '0.1.0'
 
+EXPIRE_TIME = 43200
 
 class OnionrFlow:
     def __init__(self):
@@ -75,7 +76,7 @@ class OnionrFlow:
             else:
                 if message == "q":
                     self.flowRunning = False
-                expireTime = epoch.get_epoch() + 43200
+                expireTime = epoch.get_epoch() + EXPIRE_TIME
                 if len(message) > 0:
                     logger.info('Inserting message as block...', terminal=True)
                     onionrblocks.insert(message, header='brd',
@@ -118,6 +119,24 @@ def on_flow_cmd(api, data=None):
     OnionrFlow().start()
 
 
+def on_flowsend_cmd(api, data=None):
+    err_msg = "Second arg is board name, third is quoted message"
+    try:
+        sys.argv[2]
+    except IndexError:
+        logger.error(err_msg, terminal=True)
+    try:
+        sys.argv[3]
+    except IndexError:
+        logger.error(err_msg, terminal=True)
+
+    bl = onionrblocks.insert(sys.argv[3], header='brd',
+            expire=(EXPIRE_TIME + epoch.get_epoch()),
+            meta={'ch': sys.argv[2]})
+    print(bl)
+
+
+
 def on_softreset(api, data=None):
     try:
         os.remove(identifyhome.identify_home() + '/board-index.cache.json')

static-data/default_config.json

@@ -46,7 +46,9 @@
         "minimum_score": -100
     },
     "plugins": {
-        "disabled": [],
+        "disabled": [
+            "chat"
+        ],
         "enabled": []
     },
     "timers": {

tests/integration-tests/details-test.py

@@ -0,0 +1,18 @@
+import sys
+import os
+from subprocess import Popen, PIPE
+import uuid
+
+TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
+os.environ["ONIONR_HOME"] = TEST_DIR
+
+print(f'running integration test for {__file__}')
+
+with Popen(['./onionr.sh', 'details'], stdout=PIPE) as onionr_proc:
+    output = onionr_proc.stdout.read().decode()
+if onionr_proc.returncode != 0:
+    raise ValueError('Raised non zero exit ' + str(onionr_proc.returncode))
+
+for word in ['Node', 'Human-readable']:
+    if word not in output:
+        raise ValueError(word + " not in " + output)

tests/integration-tests/export-test.py

@@ -0,0 +1,40 @@
+from unittest.mock import patch
+import sys, os
+sys.path.append(".")
+sys.path.append("src/")
+import unittest, uuid
+TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
+print("Test directory:", TEST_DIR)
+os.environ["ONIONR_HOME"] = TEST_DIR
+from utils import createdirs
+from onionrcommands import parser
+import onionrsetup as setup
+from netcontroller.torcontrol import customtorrc
+from utils import createdirs
+from onionrsetup import setup_config, setup_default_plugins
+from coredb import blockmetadb
+from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
+
+createdirs.create_dirs()
+setup_config()
+setup_default_plugins()
+import config
+from filepaths import export_location
+
+class OnionrTests(unittest.TestCase):
+    def test_export(self):
+        testargs = ["onionr.py", "flowsend", "tests", "hello"]
+        with patch.object(sys, 'argv', testargs):
+            parser.register()
+        bl = blockmetadb.get_block_list()[0]
+        testargs = ["onionr.py", "export-block", bl]
+        with patch.object(sys, 'argv', testargs):
+            parser.register()
+
+        with open(export_location + '/' + bl + BLOCK_EXPORT_FILE_EXT, 'rb') as f:
+
+            if b'hello' not in f.read():
+                raise ValueError('No exported block')
+
+
+unittest.main()

tests/integration-tests/no-command-run.py

@@ -0,0 +1,16 @@
+import sys
+import os
+from subprocess import Popen, PIPE
+import uuid
+
+TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
+os.environ["ONIONR_HOME"] = TEST_DIR
+
+print(f'running integration test for {__file__}')
+with Popen(['./onionr.sh'], stdout=PIPE) as onionr_proc:
+    output = onionr_proc.stdout.read().decode()
+if onionr_proc.returncode != 0:
+    raise ValueError('Raised non zero exit ' + str(onionr_proc.returncode))
+
+if output != '':
+    raise ValueError('No command run returned non-blank output')

tests/runtime-result.txt

@@ -1 +1 @@
-1580971981
+1581152327