improved block list syncing, added forgotten stats.js file
parent
0e6ab04996
commit
1ebed8d606
|
@ -109,7 +109,8 @@ class PublicAPI:
|
|||
|
||||
@app.route('/getblocklist')
|
||||
def getBlockList():
|
||||
bList = clientAPI._core.getBlockList()
|
||||
dateAdjust = request.args.get('date')
|
||||
bList = clientAPI._core.getBlockList(dateRec=dateAdjust)
|
||||
for b in self.hideBlocks:
|
||||
if b in bList:
|
||||
bList.remove(b)
|
||||
|
@ -304,7 +305,6 @@ class API:
|
|||
|
||||
@app.route('/queueResponseAdd/<name>', methods=['post'])
|
||||
def queueResponseAdd(name):
|
||||
print('added',name)
|
||||
self.queueResponse[name] = request.form['data']
|
||||
return Response('success')
|
||||
|
||||
|
@ -317,7 +317,6 @@ class API:
|
|||
pass
|
||||
else:
|
||||
del self.queueResponse[name]
|
||||
print(name, resp)
|
||||
return Response(resp)
|
||||
|
||||
@app.route('/ping')
|
||||
|
@ -381,6 +380,12 @@ class API:
|
|||
pass
|
||||
return Response("bye")
|
||||
|
||||
@app.route('/shutdownclean')
|
||||
def shutdownClean():
|
||||
# good for calling from other clients
|
||||
self._core.daemonQueueAdd('shutdown')
|
||||
return Response("bye")
|
||||
|
||||
@app.route('/getstats')
|
||||
def getStats():
|
||||
return Response(self._core.serializer.getStats())
|
||||
|
|
|
@ -21,7 +21,7 @@
|
|||
'''
|
||||
import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid
|
||||
import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
|
||||
import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs
|
||||
import onionrdaemontools, onionrsockets, onionr, onionrproofs, proofofmemory
|
||||
import binascii
|
||||
from dependencies import secrets
|
||||
from defusedxml import minidom
|
||||
|
@ -74,6 +74,9 @@ class OnionrCommunicatorDaemon:
|
|||
# timestamp when the last online node was seen
|
||||
self.lastNodeSeen = None
|
||||
|
||||
# Dict of time stamps for peer's block list lookup times, to avoid downloading full lists all the time
|
||||
self.dbTimestamps = {}
|
||||
|
||||
# Clear the daemon queue for any dead messages
|
||||
if os.path.exists(self._core.queueDB):
|
||||
self._core.clearDaemonQueue()
|
||||
|
@ -81,11 +84,12 @@ class OnionrCommunicatorDaemon:
|
|||
# Loads in and starts the enabled plugins
|
||||
plugins.reload()
|
||||
|
||||
self.proofofmemory = proofofmemory.ProofOfMemory(self)
|
||||
|
||||
# daemon tools are misc daemon functions, e.g. announce to online peers
|
||||
# intended only for use by OnionrCommunicatorDaemon
|
||||
self.daemonTools = onionrdaemontools.DaemonTools(self)
|
||||
|
||||
self._chat = onionrchat.OnionrChat(self)
|
||||
|
||||
if debug or developmentMode:
|
||||
OnionrCommunicatorTimers(self, self.heartbeat, 30)
|
||||
|
@ -164,7 +168,9 @@ class OnionrCommunicatorDaemon:
|
|||
existingBlocks = self._core.getBlockList()
|
||||
triedPeers = [] # list of peers we've tried this time around
|
||||
maxBacklog = 1560 # Max amount of *new* block hashes to have already in queue, to avoid memory exhaustion
|
||||
lastLookupTime = 0 # Last time we looked up a particular peer's list
|
||||
for i in range(tryAmount):
|
||||
listLookupCommand = 'getblocklist' # This is defined here to reset it each time
|
||||
if len(self.blockQueue) >= maxBacklog:
|
||||
break
|
||||
if not self.isOnline:
|
||||
|
@ -186,11 +192,21 @@ class OnionrCommunicatorDaemon:
|
|||
triedPeers.append(peer)
|
||||
if newDBHash != self._core.getAddressInfo(peer, 'DBHash'):
|
||||
self._core.setAddressInfo(peer, 'DBHash', newDBHash)
|
||||
# Get the last time we looked up a peer's stamp to only fetch blocks since then.
|
||||
# Saved in memory only for privacy reasons
|
||||
try:
|
||||
lastLookupTime = self.dbTimestamps[peer]
|
||||
except KeyError:
|
||||
lastLookupTime = 0
|
||||
else:
|
||||
listLookupCommand += '?date=%s' % (lastLookupTime,)
|
||||
try:
|
||||
newBlocks = self.peerAction(peer, 'getblocklist') # get list of new block hashes
|
||||
except Exception as error:
|
||||
logger.warn('Could not get new blocks from %s.' % peer, error = error)
|
||||
newBlocks = False
|
||||
else:
|
||||
self.dbTimestamps[peer] = self._core._utils.getRoundedEpoch(roundS=60)
|
||||
if newBlocks != False:
|
||||
# if request was a success
|
||||
for i in newBlocks.split('\n'):
|
||||
|
@ -224,8 +240,8 @@ class OnionrCommunicatorDaemon:
|
|||
if self._core._utils.storageCounter.isFull():
|
||||
break
|
||||
self.currentDownloading.append(blockHash) # So we can avoid concurrent downloading in other threads of same block
|
||||
logger.info("Attempting to download %s..." % blockHash)
|
||||
peerUsed = self.pickOnlinePeer()
|
||||
logger.info("Attempting to download %s from %s..." % (blockHash[:12], peerUsed))
|
||||
content = self.peerAction(peerUsed, 'getdata/' + blockHash) # block content from random peer (includes metadata)
|
||||
if content != False and len(content) > 0:
|
||||
try:
|
||||
|
@ -247,7 +263,7 @@ class OnionrCommunicatorDaemon:
|
|||
metadata = metas[0]
|
||||
if self._core._utils.validateMetadata(metadata, metas[2]): # check if metadata is valid, and verify nonce
|
||||
if self._core._crypto.verifyPow(content): # check if POW is enough/correct
|
||||
logger.info('Attempting to save block %s...' % blockHash)
|
||||
logger.info('Attempting to save block %s...' % blockHash[:12])
|
||||
try:
|
||||
self._core.setData(content)
|
||||
except onionrexceptions.DiskAllocationReached:
|
||||
|
@ -403,6 +419,10 @@ class OnionrCommunicatorDaemon:
|
|||
del self.connectTimes[peer]
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
del self.dbTimestamps[peer]
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
self.onlinePeers.remove(peer)
|
||||
except ValueError:
|
||||
|
|
|
@ -21,7 +21,7 @@ import sqlite3, os, sys, time, math, base64, tarfile, nacl, logger, json, netcon
|
|||
from onionrblockapi import Block
|
||||
|
||||
import onionrutils, onionrcrypto, onionrproofs, onionrevents as events, onionrexceptions, onionrvalues
|
||||
import onionrblacklist, onionrchat, onionrusers
|
||||
import onionrblacklist, onionrusers
|
||||
import dbcreator, onionrstorage, serializeddata
|
||||
|
||||
if sys.version_info < (3, 6):
|
||||
|
@ -373,11 +373,11 @@ class Core:
|
|||
try:
|
||||
c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except sqlite3.OperationalError:
|
||||
retData = False
|
||||
self.daemonQueue()
|
||||
events.event('queue_push', data = {'command': command, 'data': data}, onionr = None)
|
||||
conn.close()
|
||||
return retData
|
||||
|
||||
def daemonQueueGetResponse(self, responseID=''):
|
||||
|
@ -602,24 +602,26 @@ class Core:
|
|||
|
||||
return
|
||||
|
||||
def getBlockList(self, unsaved = False): # TODO: Use unsaved??
|
||||
def getBlockList(self, dateRec = None, unsaved = False):
|
||||
'''
|
||||
Get list of our blocks
|
||||
'''
|
||||
if dateRec == None:
|
||||
dateRec = 0
|
||||
|
||||
conn = sqlite3.connect(self.blockDB, timeout=10)
|
||||
c = conn.cursor()
|
||||
|
||||
if unsaved:
|
||||
execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
|
||||
else:
|
||||
execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;'
|
||||
|
||||
# if unsaved:
|
||||
# execute = 'SELECT hash FROM hashes WHERE dataSaved != 1 ORDER BY RANDOM();'
|
||||
# else:
|
||||
# execute = 'SELECT hash FROM hashes ORDER BY dateReceived ASC;'
|
||||
execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
|
||||
args = (dateRec,)
|
||||
rows = list()
|
||||
for row in c.execute(execute):
|
||||
for row in c.execute(execute, args):
|
||||
for i in row:
|
||||
rows.append(i)
|
||||
|
||||
return rows
|
||||
|
||||
def getBlockDate(self, blockHash):
|
||||
|
|
|
@ -21,8 +21,9 @@
|
|||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
import sys
|
||||
if sys.version_info[0] == 2 or sys.version_info[1] < 5:
|
||||
print('Error, Onionr requires Python 3.5+')
|
||||
MIN_PY_VERSION = 6
|
||||
if sys.version_info[0] == 2 or sys.version_info[1] < MIN_PY_VERSION:
|
||||
print('Error, Onionr requires Python 3.%s+' % (MIN_PY_VERSION,))
|
||||
sys.exit(1)
|
||||
import os, base64, random, getpass, shutil, subprocess, requests, time, platform, datetime, re, json, getpass, sqlite3
|
||||
import webbrowser, uuid, signal
|
||||
|
@ -198,7 +199,6 @@ class Onionr:
|
|||
|
||||
'ui' : self.openUI,
|
||||
'gui' : self.openUI,
|
||||
'chat': self.startChat,
|
||||
|
||||
'getpassword': self.printWebPassword,
|
||||
'get-password': self.printWebPassword,
|
||||
|
@ -209,8 +209,6 @@ class Onionr:
|
|||
'getpasswd': self.printWebPassword,
|
||||
'get-passwd': self.printWebPassword,
|
||||
|
||||
'chat': self.startChat,
|
||||
|
||||
'friend': self.friendCmd,
|
||||
'add-id': self.addID,
|
||||
'change-id': self.changeID
|
||||
|
@ -328,14 +326,6 @@ class Onionr:
|
|||
else:
|
||||
logger.error('Invalid key %s' % (key,))
|
||||
|
||||
def startChat(self):
|
||||
try:
|
||||
data = json.dumps({'peer': sys.argv[2], 'reason': 'chat'})
|
||||
except IndexError:
|
||||
logger.error('Must specify peer to chat with.')
|
||||
else:
|
||||
self.onionrCore.daemonQueueAdd('startSocket', data)
|
||||
|
||||
def getCommands(self):
|
||||
return self.cmds
|
||||
|
||||
|
@ -811,7 +801,7 @@ class Onionr:
|
|||
|
||||
try:
|
||||
# define stats messages here
|
||||
totalBlocks = len(Block.getBlocks())
|
||||
totalBlocks = len(self.onionrCore.getBlockList())
|
||||
signedBlocks = len(Block.getBlocks(signed = True))
|
||||
messages = {
|
||||
# info about local client
|
||||
|
|
|
@ -1,50 +0,0 @@
|
|||
'''
|
||||
Onionr - P2P Anonymous Storage Network
|
||||
|
||||
Onionr Chat Messages
|
||||
'''
|
||||
'''
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
import logger, time
|
||||
|
||||
class OnionrChat:
|
||||
def __init__(self, communicatorInst):
|
||||
'''OnionrChat uses onionrsockets (handled by the communicator) to exchange direct chat messages'''
|
||||
self.communicator = communicatorInst
|
||||
self._core = self.communicator._core
|
||||
self._utils = self._core._utils
|
||||
|
||||
self.chats = {} # {'peer': {'date': date, message': message}}
|
||||
self.chatSend = {}
|
||||
|
||||
def chatHandler(self):
|
||||
while not self.communicator.shutdown:
|
||||
for peer in self._core.socketServerConnData:
|
||||
try:
|
||||
assert self._core.socketReasons[peer] == "chat"
|
||||
except (AssertionError, KeyError) as e:
|
||||
logger.warn('Peer is not for chat')
|
||||
continue
|
||||
else:
|
||||
self.chats[peer] = {'date': self._core.socketServerConnData[peer]['date'], 'data': self._core.socketServerConnData[peer]['data']}
|
||||
logger.info("CHAT MESSAGE RECIEVED: %s" % self.chats[peer]['data'])
|
||||
for peer in self.communicator.socketClient.sockets:
|
||||
try:
|
||||
logger.info(self.communicator.socketClient.connPool[peer]['data'])
|
||||
self.communicator.socketClient.sendData(peer, "lol")
|
||||
except:
|
||||
pass
|
||||
|
||||
time.sleep(2)
|
|
@ -183,7 +183,6 @@ class OnionrUtils:
|
|||
if data != '':
|
||||
data = '&data=' + urllib.parse.quote_plus(data)
|
||||
payload = 'http://%s/%s%s' % (hostname, command, data)
|
||||
print(payload,config.get('client.webpassword'))
|
||||
try:
|
||||
if post:
|
||||
retData = requests.post(payload, data=postData, headers={'token': config.get('client.webpassword')}, timeout=(15, 30)).text
|
||||
|
|
|
@ -2,8 +2,8 @@
|
|||
"general" : {
|
||||
"dev_mode" : true,
|
||||
"display_header" : false,
|
||||
"minimum_block_pow": 3,
|
||||
"minimum_send_pow": 3,
|
||||
"minimum_block_pow": 1,
|
||||
"minimum_send_pow": 1,
|
||||
"socket_servers": false,
|
||||
"security_level": 0,
|
||||
"max_block_age": 2678400,
|
||||
|
|
|
@ -5,20 +5,27 @@
|
|||
<title>
|
||||
Onionr
|
||||
</title>
|
||||
<link rel='stylesheet' href='/shared/style/modal.css'>
|
||||
<link rel='stylesheet' href='/shared/main/style.css'>
|
||||
</head>
|
||||
<body>
|
||||
<div id="shutdownNotice" class='overlay'>
|
||||
<div>
|
||||
<p>Your node will shutdown. Thank you for using Onionr.</p>
|
||||
</div>
|
||||
</div>
|
||||
<img class='logo' src='/shared/onionr-icon.png' alt='onionr logo'>
|
||||
<span class='logoText'>Onionr Web Control Panel</span>
|
||||
<div class='content'>
|
||||
<a href='shutdown'>Shutdown node</a>
|
||||
<button id='shutdownNode'>Shutdown Node</button>
|
||||
<h2>Stats</h2>
|
||||
<p>Uptime: <span id='uptime'></span></p>
|
||||
<p>Stored Blocks: <span id='storedBlocks'></span></p>
|
||||
<p>Connected nodes:</p>
|
||||
<pre id='connectedNodes'></pre>
|
||||
</div>
|
||||
<script src='/shared/misc.js'></script>
|
||||
<script src='/shared/main/stats.js'></script>
|
||||
<script src='/shared/panel.js'></script>
|
||||
</body>
|
||||
<script src='/shared/misc.js'></script>
|
||||
<script src='/shared/main/stats.js'></script>
|
||||
</html>
|
|
@ -17,14 +17,11 @@
|
|||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>
|
||||
*/
|
||||
|
||||
uptimeDisplay = document.getElementById('uptime')
|
||||
connectedDisplay = document.getElementById('connectedNodes')
|
||||
storedBlockDisplay = document.getElementById('storedBlocks')
|
||||
|
||||
pass = window.location.hash.replace('#', '')
|
||||
|
||||
stats = JSON.parse(httpGet('getstats', pass))
|
||||
stats = JSON.parse(httpGet('getstats', webpass))
|
||||
uptimeDisplay.innerText = stats['uptime'] + ' seconds'
|
||||
connectedDisplay.innerText = stats['connectedNodes']
|
||||
storedBlockDisplay.innerText = stats['blockCount']
|
|
@ -124,3 +124,17 @@ body{
|
|||
display: block;
|
||||
}
|
||||
}
|
||||
|
||||
/*https://stackoverflow.com/a/16778646*/
|
||||
.overlay {
|
||||
visibility: hidden;
|
||||
position: absolute;
|
||||
left: 0px;
|
||||
top: 0px;
|
||||
width:100%;
|
||||
opacity: 0.9;
|
||||
height:100%;
|
||||
text-align:center;
|
||||
z-index: 1000;
|
||||
background-color: black;
|
||||
}
|
||||
|
|
|
@ -1,4 +1,16 @@
|
|||
function httpGet(theUrl, webpass) {
|
||||
webpass = document.location.hash.replace('#', '')
|
||||
if (typeof webpass == "undefined"){
|
||||
webpass = localStorage['webpass']
|
||||
}
|
||||
else{
|
||||
localStorage['webpass'] = webpass
|
||||
document.location.hash = ''
|
||||
}
|
||||
if (typeof webpass == "undefined" || webpass == ""){
|
||||
alert('Web password was not found in memory or URL')
|
||||
}
|
||||
|
||||
function httpGet(theUrl) {
|
||||
var xmlHttp = new XMLHttpRequest()
|
||||
xmlHttp.open( "GET", theUrl, false ) // false for synchronous request
|
||||
xmlHttp.setRequestHeader('token', webpass)
|
||||
|
@ -7,6 +19,10 @@ function httpGet(theUrl, webpass) {
|
|||
return xmlHttp.responseText
|
||||
}
|
||||
else{
|
||||
return "";
|
||||
return ""
|
||||
}
|
||||
}
|
||||
function overlay(overlayID) {
|
||||
el = document.getElementById(overlayID)
|
||||
el.style.visibility = (el.style.visibility == "visible") ? "hidden" : "visible"
|
||||
}
|
||||
|
|
|
@ -0,0 +1,6 @@
|
|||
shutdownBtn = document.getElementById('shutdownNode')
|
||||
|
||||
shutdownBtn.onclick = function(){
|
||||
httpGet('shutdownclean')
|
||||
overlay('shutdownNotice')
|
||||
}
|
Loading…
Reference in New Issue