work on new storage system

onionr/apimanager.py

@@ -1,75 +0,0 @@
-'''
-    Onionr - P2P Anonymous Storage Network
-
-    Handles api data exchange, interfaced by both public and client http api
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import config, apipublic, apiprivate, core, socket, random, threading, time
-config.reload()
-
-PRIVATE_API_VERSION = 0
-PUBLIC_API_VERSION = 1
-
-DEV_MODE = config.get('general.dev_mode')
-
-def getOpenPort():
-    '''Get a random open port'''
-    p = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    p.bind(("127.0.0.1",0))
-    p.listen(1)
-    port = p.getsockname()[1]
-    p.close()
-    return port
-
-def getRandomLocalIP():
-    '''Get a random local ip address'''
-    hostOctets = [str(127), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF)), str(random.randint(0x02, 0xFF))]
-    host = '.'.join(hostOctets)
-    return host
-
-class APIManager:
-    def __init__(self, coreInst):
-        assert isinstance(coreInst, core.Core)
-        self.core = coreInst
-        self.utils = coreInst._utils
-        self.crypto = coreInst._crypto
-        
-        # if this gets set to true, both the public and private apis will shutdown
-        self.shutdown = False
-        
-        publicIP = '127.0.0.1'
-        privateIP = '127.0.0.1'
-        if DEV_MODE:
-            # set private and local api servers bind IPs to random localhost (127.x.x.x), make sure not the same
-            privateIP = getRandomLocalIP()
-            while True:
-                publicIP = getRandomLocalIP()
-                if publicIP != privateIP:
-                    break
-        
-        # Make official the IPs and Ports
-        self.publicIP = publicIP
-        self.privateIP = privateIP
-        self.publicPort = config.get('client.port', 59496)
-        self.privatePort = config.get('client.port', 59496)
-
-        # Run the API servers in new threads
-        self.publicAPI = apipublic.APIPublic(self)
-        self.privateAPI = apiprivate.APIPrivate(self)
-        threading.Thread(target=self.publicAPI.run).start()
-        threading.Thread(target=self.privateAPI.run).start()
-        while not self.shutdown:
-            time.sleep(1)

onionr/apiprivate.py

@@ -1,32 +0,0 @@
-'''
-    Onionr - P2P Anonymous Storage Network
-
-    Handle incoming commands from the client. Intended for localhost use
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import flask, apimanager
-from flask import request, Response, abort, send_from_directory
-from gevent.pywsgi import WSGIServer
-
-class APIPrivate:
-    def __init__(self, managerInst):
-        assert isinstance(managerInst, apimanager.APIManager)
-        self.app = flask.Flask(__name__) # The flask application, which recieves data from the greenlet wsgiserver
-        self.httpServer = WSGIServer((managerInst.privateIP, managerInst.privatePort), self.app, log=None)
-
-    def run(self):
-        self.httpServer.serve_forever()
-        return

onionr/apipublic.py

@@ -1,41 +0,0 @@
-'''
-    Onionr - P2P Anonymous Storage Network
-
-    Handle incoming commands from other Onionr nodes, over HTTP
-'''
-'''
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-'''
-import flask, apimanager
-from flask import request, Response, abort, send_from_directory
-from gevent.pywsgi import WSGIServer
-
-
-class APIPublic:
-    def __init__(self, managerInst):
-        assert isinstance(managerInst, apimanager.APIManager)
-        app = flask.Flask(__name__)
-        @app.route('/')
-        def banner():
-            try:
-                with open('static-data/index.html', 'r') as html:
-                    resp = Response(html.read(), mimetype='text/html')
-            except FileNotFoundError:
-                resp = Response("")
-            return resp
-        self.httpServer = WSGIServer((managerInst.publicIP, managerInst.publicPort), app)
-
-    def run(self):
-        self.httpServer.serve_forever()
-        return

onionr/core.py

@@ -48,8 +48,8 @@ class Core:
             self.queueDB = self.dataDir + 'queue.db'
             self.peerDB = self.dataDir + 'peers.db'
             self.blockDB = self.dataDir + 'blocks.db'
-            self.blockDataDB = self.dataDir + 'block-data.db'
             self.blockDataLocation = self.dataDir + 'blocks/'
+            self.blockDataDB = self.blockDataLocation + 'block-data.db'
             self.publicApiHostFile = self.dataDir + 'public-host.txt'
             self.privateApiHostFile = self.dataDir + 'private-host.txt'
             self.addressDB = self.dataDir + 'address.db'

onionr/onionrstorage.py

@@ -17,15 +17,51 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 '''
-import core
+import core, sys, sqlite3, os
-class OnionrStorage:
-    def __init__(self, coreInst):
-        assert isinstance(coreInst, core.Core)
-        self._core = coreInst
-        return
 
-    def store(self, hash, data):
+DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
-        return
 
-    def getData(self, hash):
+def _dbInsert(coreInst, blockHash, data):
-        return
+    assert isinstance(core, core.Core)
+    conn = sqlite3.connect(coreInst.blockDataDB, timeout=10)
+    c = conn.cursor()
+    data = (blockHash, data)
+    c.execute('INSERT INTO blockData (hash, data) VALUES(?, ?);', data)
+    conn.commit()
+    conn.close()
+
+def _dbFetch(coreInst, blockHash):
+    conn = sqlite3.connect(coreInst.blockDataDB, timeout=10)
+    c = conn.cursor()
+    for i in c.execute('SELECT data from blockData where hash = ?', (blockHash,)):
+        return i[0]
+    conn.commit()
+    conn.close()
+    return None
+
+def store(coreInst, blockHash, data):
+    assert isinstance(coreInst, core.Core)
+    assert self._core._utils.validateHash(blockHash)
+    assert self._core._crypto.sha3Hash(data) == blockHash
+    
+    if DB_ENTRY_SIZE_LIMIT >= sys.getsizeof(data):
+        _dbInsert(coreInst, blockHash, data)
+    else:
+        with open('%s/%s.dat' % (coreInst.blockDataLocation, blockHash), 'w') as blockFile:
+            blockFile.write(data)
+
+def getData(coreInst, bHash):
+    assert isinstance(coreInst, core.Core)
+    assert self._core._utils.validateHash(blockHash)
+
+    # First check DB for data entry by hash
+    # if no entry, check disk
+    # If no entry in either, raise an exception
+    retData = ''
+    fileLocation = '%s/%s.dat' % (coreInst.blockDataLocation, bHash)
+    if os.path.exists(fileLocation):
+        with open(fileLocation, 'r') as block:
+            retData = block.read()
+    else:
+        retData = _dbFetch(coreInst, bHash)
+    return

onionr/static-data/www/ui/dist/js/main.js

@@ -704,7 +704,7 @@ if(tt !== null && tt !== undefined) {
 if(getWebPassword() === null) {
     var password = "";
     while(password.length != 64) {
-        password = prompt("Please enter the web password (run `./RUN-LINUX.sh --get-password`)");
+        password = prompt("Please enter the web password (run `./RUN-LINUX.sh --details`)");
     }
 
     setWebPassword(password);