From 1ebed8d6068e40adf2c5fd82d5029002cf7ad4ee Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Tue, 15 Jan 2019 23:57:47 -0600 Subject: [PATCH] improved block list syncing, added forgotten stats.js file --- onionr/api.py | 11 +++-- onionr/communicator2.py | 28 +++++++++-- onionr/core.py | 22 +++++---- onionr/onionr.py | 18 ++----- onionr/onionrchat.py | 50 -------------------- onionr/onionrutils.py | 1 - onionr/static-data/default_config.json | 4 +- onionr/static-data/www/private/index.html | 13 +++-- onionr/static-data/www/shared/main/stats.js | 5 +- onionr/static-data/www/shared/main/style.css | 14 ++++++ onionr/static-data/www/shared/misc.js | 22 +++++++-- onionr/static-data/www/shared/panel.js | 6 +++ 12 files changed, 100 insertions(+), 94 deletions(-) delete mode 100644 onionr/onionrchat.py create mode 100644 onionr/static-data/www/shared/panel.js diff --git a/onionr/api.py b/onionr/api.py index 358b64ad..ab8438bd 100755 --- a/onionr/api.py +++ b/onionr/api.py @@ -109,7 +109,8 @@ class PublicAPI: @app.route('/getblocklist') def getBlockList(): - bList = clientAPI._core.getBlockList() + dateAdjust = request.args.get('date') + bList = clientAPI._core.getBlockList(dateRec=dateAdjust) for b in self.hideBlocks: if b in bList: bList.remove(b) @@ -304,7 +305,6 @@ class API: @app.route('/queueResponseAdd/', methods=['post']) def queueResponseAdd(name): - print('added',name) self.queueResponse[name] = request.form['data'] return Response('success') @@ -317,7 +317,6 @@ class API: pass else: del self.queueResponse[name] - print(name, resp) return Response(resp) @app.route('/ping') @@ -380,6 +379,12 @@ class API: except AttributeError: pass return Response("bye") + + @app.route('/shutdownclean') + def shutdownClean(): + # good for calling from other clients + self._core.daemonQueueAdd('shutdown') + return Response("bye") @app.route('/getstats') def getStats(): diff --git a/onionr/communicator2.py b/onionr/communicator2.py index 95c84b32..70c32988 100755 --- a/onionr/communicator2.py +++ b/onionr/communicator2.py @@ -21,7 +21,7 @@ ''' import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block -import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs +import onionrdaemontools, onionrsockets, onionr, onionrproofs, proofofmemory import binascii from dependencies import secrets from defusedxml import minidom @@ -74,6 +74,9 @@ class OnionrCommunicatorDaemon: # timestamp when the last online node was seen self.lastNodeSeen = None + # Dict of time stamps for peer's block list lookup times, to avoid downloading full lists all the time + self.dbTimestamps = {} + # Clear the daemon queue for any dead messages if os.path.exists(self._core.queueDB): self._core.clearDaemonQueue() @@ -81,11 +84,12 @@ class OnionrCommunicatorDaemon: # Loads in and starts the enabled plugins plugins.reload() + self.proofofmemory = proofofmemory.ProofOfMemory(self) + # daemon tools are misc daemon functions, e.g. announce to online peers # intended only for use by OnionrCommunicatorDaemon self.daemonTools = onionrdaemontools.DaemonTools(self) - self._chat = onionrchat.OnionrChat(self) if debug or developmentMode: OnionrCommunicatorTimers(self, self.heartbeat, 30) @@ -164,7 +168,9 @@ class OnionrCommunicatorDaemon: existingBlocks = self._core.getBlockList() triedPeers = [] # list of peers we've tried this time around maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion + lastLookupTime = 0 # Last time we looked up a particular peer's list for i in range(tryAmount): + listLookupCommand = 'getblocklist' # This is defined here to reset it each time if len(self.blockQueue) >= maxBacklog: break if not self.isOnline: @@ -186,11 +192,21 @@ class OnionrCommunicatorDaemon: triedPeers.append(peer) if newDBHash != self._core.getAddressInfo(peer, 'DBHash'): self._core.setAddressInfo(peer, 'DBHash', newDBHash) + # Get the last time we looked up a peer's stamp to only fetch blocks since then. + # Saved in memory only for privacy reasons + try: + lastLookupTime = self.dbTimestamps[peer] + except KeyError: + lastLookupTime = 0 + else: + listLookupCommand += '?date=%s' % (lastLookupTime,) try: newBlocks = self.peerAction(peer, 'getblocklist') # get list of new block hashes except Exception as error: logger.warn('Could not get new blocks from %s.' % peer, error = error) newBlocks = False + else: + self.dbTimestamps[peer] = self._core._utils.getRoundedEpoch(roundS=60) if newBlocks != False: # if request was a success for i in newBlocks.split('\n'): @@ -224,8 +240,8 @@ class OnionrCommunicatorDaemon: if self._core._utils.storageCounter.isFull(): break self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block - logger.info("Attempting to download %s..." % blockHash) peerUsed = self.pickOnlinePeer() + logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed)) content = self.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata) if content != False and len(content) > 0: try: @@ -247,7 +263,7 @@ class OnionrCommunicatorDaemon: metadata = metas[0] if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce if self._core._crypto.verifyPow(content): # check if POW is enough/correct - logger.info('Attempting to save block %s...' % blockHash) + logger.info('Attempting to save block %s...' % blockHash[:12]) try: self._core.setData(content) except onionrexceptions.DiskAllocationReached: @@ -403,6 +419,10 @@ class OnionrCommunicatorDaemon: del self.connectTimes[peer] except KeyError: pass + try: + del self.dbTimestamps[peer] + except KeyError: + pass try: self.onlinePeers.remove(peer) except ValueError: diff --git a/onionr/core.py b/onionr/core.py index 987a6940..b2ec0dc6 100644 --- a/onionr/core.py +++ b/onionr/core.py @@ -21,7 +21,7 @@ import sqlite3, os, sys, time, math, base64, tarfile, nacl, logger, json, netcon from onionrblockapi import Block import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues -import onionrblacklist, onionrchat, onionrusers +import onionrblacklist, onionrusers import dbcreator, onionrstorage, serializeddata if sys.version_info < (3, 6): @@ -373,11 +373,11 @@ class Core: try: c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t) conn.commit() - conn.close() except sqlite3.OperationalError: retData = False self.daemonQueue() events.event('queue_push', data = {'command': command, 'data': data}, onionr = None) + conn.close() return retData def daemonQueueGetResponse(self, responseID=''): @@ -602,24 +602,26 @@ class Core: return - def getBlockList(self, unsaved = False): # TODO: Use unsaved?? + def getBlockList(self, dateRec = None, unsaved = False): ''' Get list of our blocks ''' + if dateRec == None: + dateRec = 0 conn = sqlite3.connect(self.blockDB, timeout=10) c = conn.cursor() - if unsaved: - execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();' - else: - execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;' - + # if unsaved: + # execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();' + # else: + # execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;' + execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;' + args = (dateRec,) rows = list() - for row in c.execute(execute): + for row in c.execute(execute, args): for i in row: rows.append(i) - return rows def getBlockDate(self, blockHash): diff --git a/onionr/onionr.py b/onionr/onionr.py index c213c0f9..e54e90bb 100755 --- a/onionr/onionr.py +++ b/onionr/onionr.py @@ -21,8 +21,9 @@ along with this program. If not, see . ''' import sys -if sys.version_info[0] == 2 or sys.version_info[1] < 5: - print('Error, Onionr requires Python 3.5+') +MIN_PY_VERSION = 6 +if sys.version_info[0] == 2 or sys.version_info[1] < MIN_PY_VERSION: + print('Error, Onionr requires Python 3.%s+' % (MIN_PY_VERSION,)) sys.exit(1) import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass, sqlite3 import webbrowser, uuid, signal @@ -198,7 +199,6 @@ class Onionr: 'ui' : self.openUI, 'gui' : self.openUI, - 'chat': self.startChat, 'getpassword': self.printWebPassword, 'get-password': self.printWebPassword, @@ -209,8 +209,6 @@ class Onionr: 'getpasswd': self.printWebPassword, 'get-passwd': self.printWebPassword, - 'chat': self.startChat, - 'friend': self.friendCmd, 'add-id': self.addID, 'change-id': self.changeID @@ -328,14 +326,6 @@ class Onionr: else: logger.error('Invalid key %s' % (key,)) - def startChat(self): - try: - data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'}) - except IndexError: - logger.error('Must specify peer to chat with.') - else: - self.onionrCore.daemonQueueAdd('startSocket', data) - def getCommands(self): return self.cmds @@ -811,7 +801,7 @@ class Onionr: try: # define stats messages here - totalBlocks = len(Block.getBlocks()) + totalBlocks = len(self.onionrCore.getBlockList()) signedBlocks = len(Block.getBlocks(signed = True)) messages = { # info about local client diff --git a/onionr/onionrchat.py b/onionr/onionrchat.py deleted file mode 100644 index 84483295..00000000 --- a/onionr/onionrchat.py +++ /dev/null @@ -1,50 +0,0 @@ -''' - Onionr - P2P Anonymous Storage Network - - Onionr Chat Messages -''' -''' - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -''' -import logger, time - -class OnionrChat: - def __init__(self, communicatorInst): - '''OnionrChat uses onionrsockets (handled by the communicator) to exchange direct chat messages''' - self.communicator = communicatorInst - self._core = self.communicator._core - self._utils = self._core._utils - - self.chats = {} # {'peer': {'date': date, message': message}} - self.chatSend = {} - - def chatHandler(self): - while not self.communicator.shutdown: - for peer in self._core.socketServerConnData: - try: - assert self._core.socketReasons[peer] == "chat" - except (AssertionError, KeyError) as e: - logger.warn('Peer is not for chat') - continue - else: - self.chats[peer] = {'date': self._core.socketServerConnData[peer]['date'], 'data': self._core.socketServerConnData[peer]['data']} - logger.info("CHAT MESSAGE RECIEVED: %s" % self.chats[peer]['data']) - for peer in self.communicator.socketClient.sockets: - try: - logger.info(self.communicator.socketClient.connPool[peer]['data']) - self.communicator.socketClient.sendData(peer, "lol") - except: - pass - - time.sleep(2) diff --git a/onionr/onionrutils.py b/onionr/onionrutils.py index 7207756e..332d6a70 100644 --- a/onionr/onionrutils.py +++ b/onionr/onionrutils.py @@ -183,7 +183,6 @@ class OnionrUtils: if data != '': data = '&data=' + urllib.parse.quote_plus(data) payload = 'http://%s/%s%s' % (hostname, command, data) - print(payload,config.get('client.webpassword')) try: if post: retData = requests.post(payload, data=postData, headers={'token': config.get('client.webpassword')}, timeout=(15, 30)).text diff --git a/onionr/static-data/default_config.json b/onionr/static-data/default_config.json index ed9270db..2ef339bc 100644 --- a/onionr/static-data/default_config.json +++ b/onionr/static-data/default_config.json @@ -2,8 +2,8 @@ "general" : { "dev_mode" : true, "display_header" : false, - "minimum_block_pow": 3, - "minimum_send_pow": 3, + "minimum_block_pow": 1, + "minimum_send_pow": 1, "socket_servers": false, "security_level": 0, "max_block_age": 2678400, diff --git a/onionr/static-data/www/private/index.html b/onionr/static-data/www/private/index.html index 58c14e3f..12f05b85 100644 --- a/onionr/static-data/www/private/index.html +++ b/onionr/static-data/www/private/index.html @@ -5,20 +5,27 @@ Onionr + +
+
+

Your node will shutdown. Thank you for using Onionr.

+
+
Onionr Web Control Panel
- Shutdown node +

Stats

Uptime:

Stored Blocks:

Connected nodes:


         

+        
+        
+        
     
-    
-    
 
\ No newline at end of file
diff --git a/onionr/static-data/www/shared/main/stats.js b/onionr/static-data/www/shared/main/stats.js
index 4e856e5a..6796f2b7 100644
--- a/onionr/static-data/www/shared/main/stats.js
+++ b/onionr/static-data/www/shared/main/stats.js
@@ -17,14 +17,11 @@
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see 
 */
-
 uptimeDisplay = document.getElementById('uptime')
 connectedDisplay = document.getElementById('connectedNodes')
 storedBlockDisplay = document.getElementById('storedBlocks')
 
-pass = window.location.hash.replace('#', '')
-
-stats = JSON.parse(httpGet('getstats', pass))
+stats = JSON.parse(httpGet('getstats', webpass))
 uptimeDisplay.innerText = stats['uptime'] + ' seconds'
 connectedDisplay.innerText = stats['connectedNodes']
 storedBlockDisplay.innerText = stats['blockCount']
\ No newline at end of file
diff --git a/onionr/static-data/www/shared/main/style.css b/onionr/static-data/www/shared/main/style.css
index 7b8f3c2e..69e1a407 100644
--- a/onionr/static-data/www/shared/main/style.css
+++ b/onionr/static-data/www/shared/main/style.css
@@ -124,3 +124,17 @@ body{
         display: block;
     }
 }
+
+/*https://stackoverflow.com/a/16778646*/
+.overlay {
+    visibility: hidden;
+    position: absolute;
+    left: 0px;
+    top: 0px;
+    width:100%;
+    opacity: 0.9;
+    height:100%;
+    text-align:center;
+    z-index: 1000;
+    background-color: black;
+ }
diff --git a/onionr/static-data/www/shared/misc.js b/onionr/static-data/www/shared/misc.js
index 1b7d6092..ae0c1fb0 100644
--- a/onionr/static-data/www/shared/misc.js
+++ b/onionr/static-data/www/shared/misc.js
@@ -1,4 +1,16 @@
-function httpGet(theUrl, webpass) {
+webpass = document.location.hash.replace('#', '')
+if (typeof webpass == "undefined"){
+    webpass = localStorage['webpass']
+}
+else{
+    localStorage['webpass'] = webpass
+    document.location.hash = ''
+}
+if (typeof webpass == "undefined" || webpass == ""){
+    alert('Web password was not found in memory or URL')
+}
+
+function httpGet(theUrl) {
     var xmlHttp = new XMLHttpRequest()
     xmlHttp.open( "GET", theUrl, false ) // false for synchronous request
     xmlHttp.setRequestHeader('token', webpass)
@@ -7,6 +19,10 @@ function httpGet(theUrl, webpass) {
         return xmlHttp.responseText
     }
     else{
-        return "";
+        return ""
     }
-}
\ No newline at end of file
+}
+function overlay(overlayID) {
+    el = document.getElementById(overlayID)
+   el.style.visibility = (el.style.visibility == "visible") ? "hidden" : "visible"
+ }
diff --git a/onionr/static-data/www/shared/panel.js b/onionr/static-data/www/shared/panel.js
new file mode 100644
index 00000000..c79241c0
--- /dev/null
+++ b/onionr/static-data/www/shared/panel.js
@@ -0,0 +1,6 @@
+shutdownBtn = document.getElementById('shutdownNode')
+
+shutdownBtn.onclick = function(){
+    httpGet('shutdownclean')
+    overlay('shutdownNotice')
+}