+ added reverse block insertion
* handle downloading of blocks better when peer goes offline * bumped default disk allocation * added post request utilmaster
parent
8e1b6e1e7e
commit
71007a2d0a
|
@ -24,7 +24,7 @@ from gevent.wsgi import WSGIServer
|
||||||
import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config
|
import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config
|
||||||
from core import Core
|
from core import Core
|
||||||
from onionrblockapi import Block
|
from onionrblockapi import Block
|
||||||
import onionrutils, onionrcrypto
|
import onionrutils, onionrcrypto, blockimporter
|
||||||
|
|
||||||
class API:
|
class API:
|
||||||
'''
|
'''
|
||||||
|
@ -141,9 +141,6 @@ class API:
|
||||||
resp = Response('Goodbye')
|
resp = Response('Goodbye')
|
||||||
elif action == 'ping':
|
elif action == 'ping':
|
||||||
resp = Response('pong')
|
resp = Response('pong')
|
||||||
elif action == 'stats':
|
|
||||||
resp = Response('me_irl')
|
|
||||||
raise Exception
|
|
||||||
elif action == 'site':
|
elif action == 'site':
|
||||||
block = data
|
block = data
|
||||||
siteData = self._core.getData(data)
|
siteData = self._core.getData(data)
|
||||||
|
@ -175,6 +172,24 @@ class API:
|
||||||
resp = Response("")
|
resp = Response("")
|
||||||
return resp
|
return resp
|
||||||
|
|
||||||
|
@app.route('/public/upload/', methods=['POST'])
|
||||||
|
def blockUpload():
|
||||||
|
self.validateHost('public')
|
||||||
|
resp = 'failure'
|
||||||
|
try:
|
||||||
|
data = request.form['block']
|
||||||
|
except KeyError:
|
||||||
|
logger.warn('No block specified for upload')
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
if sys.getsizeof(data) < 100000000:
|
||||||
|
if blockimporter.importBlockFromData(data, self._core):
|
||||||
|
resp = 'success'
|
||||||
|
else:
|
||||||
|
logger.warn('Error encountered importing uploaded block')
|
||||||
|
|
||||||
|
resp = Response(resp)
|
||||||
|
return resp
|
||||||
@app.route('/public/')
|
@app.route('/public/')
|
||||||
def public_handler():
|
def public_handler():
|
||||||
# Public means it is publicly network accessible
|
# Public means it is publicly network accessible
|
||||||
|
@ -198,7 +213,6 @@ class API:
|
||||||
resp = Response('\n'.join(self._core.getBlockList()))
|
resp = Response('\n'.join(self._core.getBlockList()))
|
||||||
elif action == 'directMessage':
|
elif action == 'directMessage':
|
||||||
resp = Response(self._core.handle_direct_connection(data))
|
resp = Response(self._core.handle_direct_connection(data))
|
||||||
#elif action == 'nodeProof':
|
|
||||||
|
|
||||||
elif action == 'announce':
|
elif action == 'announce':
|
||||||
if data != '':
|
if data != '':
|
||||||
|
|
|
@ -36,6 +36,8 @@ class OnionrCommunicatorDaemon:
|
||||||
# intalize NIST beacon salt and time
|
# intalize NIST beacon salt and time
|
||||||
self.nistSaltTimestamp = 0
|
self.nistSaltTimestamp = 0
|
||||||
self.powSalt = 0
|
self.powSalt = 0
|
||||||
|
|
||||||
|
self.blockToUpload = ''
|
||||||
|
|
||||||
# loop time.sleep delay in seconds
|
# loop time.sleep delay in seconds
|
||||||
self.delay = 1
|
self.delay = 1
|
||||||
|
@ -84,7 +86,7 @@ class OnionrCommunicatorDaemon:
|
||||||
OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
|
OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
|
||||||
|
|
||||||
# set loop to execute instantly to load up peer pool (replaced old pool init wait)
|
# set loop to execute instantly to load up peer pool (replaced old pool init wait)
|
||||||
peerPoolTimer.count = (peerPoolTimer.frequency - 1)
|
peerPoolTimer.count = (peerPoolTimer.frequency - 1)
|
||||||
|
|
||||||
# Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
|
# Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
|
||||||
try:
|
try:
|
||||||
|
@ -101,7 +103,7 @@ class OnionrCommunicatorDaemon:
|
||||||
logger.info('Goodbye.')
|
logger.info('Goodbye.')
|
||||||
self._core._utils.localCommand('shutdown')
|
self._core._utils.localCommand('shutdown')
|
||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
|
|
||||||
def lookupKeys(self):
|
def lookupKeys(self):
|
||||||
'''Lookup new keys'''
|
'''Lookup new keys'''
|
||||||
logger.debug('Looking up new keys...')
|
logger.debug('Looking up new keys...')
|
||||||
|
@ -111,7 +113,6 @@ class OnionrCommunicatorDaemon:
|
||||||
peer = self.pickOnlinePeer()
|
peer = self.pickOnlinePeer()
|
||||||
newKeys = self.peerAction(peer, action='kex')
|
newKeys = self.peerAction(peer, action='kex')
|
||||||
self._core._utils.mergeKeys(newKeys)
|
self._core._utils.mergeKeys(newKeys)
|
||||||
|
|
||||||
self.decrementThreadCount('lookupKeys')
|
self.decrementThreadCount('lookupKeys')
|
||||||
return
|
return
|
||||||
|
|
||||||
|
@ -196,7 +197,7 @@ class OnionrCommunicatorDaemon:
|
||||||
pass
|
pass
|
||||||
logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
|
logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
|
||||||
self.blockQueue.remove(blockHash) # remove from block queue both if success or false
|
self.blockQueue.remove(blockHash) # remove from block queue both if success or false
|
||||||
self.currentDownloading.remove(blockHash)
|
self.currentDownloading.remove(blockHash)
|
||||||
self.decrementThreadCount('getBlocks')
|
self.decrementThreadCount('getBlocks')
|
||||||
return
|
return
|
||||||
|
|
||||||
|
@ -339,10 +340,31 @@ class OnionrCommunicatorDaemon:
|
||||||
for i in self.timers:
|
for i in self.timers:
|
||||||
if i.timerFunction.__name__ == 'lookupKeys':
|
if i.timerFunction.__name__ == 'lookupKeys':
|
||||||
i.count = (i.frequency - 1)
|
i.count = (i.frequency - 1)
|
||||||
|
elif cmd[0] == 'uploadBlock':
|
||||||
|
self.blockToUpload = cmd[1]
|
||||||
|
threading.Thread(target=self.uploadBlock).start()
|
||||||
else:
|
else:
|
||||||
logger.info('Recieved daemonQueue command:' + cmd[0])
|
logger.info('Recieved daemonQueue command:' + cmd[0])
|
||||||
self.decrementThreadCount('daemonCommands')
|
self.decrementThreadCount('daemonCommands')
|
||||||
|
|
||||||
|
def uploadBlock(self):
|
||||||
|
tiredPeers = []
|
||||||
|
if not self._core._utils.validateHash(self.blockToUpload):
|
||||||
|
logger.warn('Requested to upload invalid block')
|
||||||
|
return
|
||||||
|
for i in max(len(self.onlinePeers), 2):
|
||||||
|
while True:
|
||||||
|
peer = self.pickOnlinePeer()
|
||||||
|
if peer
|
||||||
|
url = 'http://' + peer + '/public/upload/'
|
||||||
|
data = {'block': block.Block(self.blockToUpload).getRaw()}
|
||||||
|
if peer.endswith('.onion'):
|
||||||
|
proxyType = 'tor'
|
||||||
|
elif peer.endswith('.i2p'):
|
||||||
|
proxyType = 'i2p'
|
||||||
|
logger.info("Uploading block")
|
||||||
|
self._core._utils.doPostRequest(url, data=data, proxyType=proxyType)
|
||||||
|
|
||||||
def announce(self, peer):
|
def announce(self, peer):
|
||||||
'''Announce to peers our address'''
|
'''Announce to peers our address'''
|
||||||
announceCount = 0
|
announceCount = 0
|
||||||
|
|
|
@ -41,10 +41,12 @@ class Core:
|
||||||
self.blockDataLocation = 'data/blocks/'
|
self.blockDataLocation = 'data/blocks/'
|
||||||
self.addressDB = 'data/address.db'
|
self.addressDB = 'data/address.db'
|
||||||
self.hsAdder = ''
|
self.hsAdder = ''
|
||||||
|
|
||||||
self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
|
self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
|
||||||
self.bootstrapList = []
|
self.bootstrapList = []
|
||||||
self.requirements = onionrvalues.OnionrValues()
|
self.requirements = onionrvalues.OnionrValues()
|
||||||
|
self.torPort = torPort
|
||||||
|
|
||||||
|
self.usageFile = 'data/disk-usage.txt'
|
||||||
|
|
||||||
if not os.path.exists('data/'):
|
if not os.path.exists('data/'):
|
||||||
os.mkdir('data/')
|
os.mkdir('data/')
|
||||||
|
@ -757,6 +759,7 @@ class Core:
|
||||||
retData = self.setData(payload)
|
retData = self.setData(payload)
|
||||||
self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
|
self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
|
||||||
self.setBlockType(retData, meta['type'])
|
self.setBlockType(retData, meta['type'])
|
||||||
|
self.daemonQueueAdd('uploadBlock', retData)
|
||||||
|
|
||||||
if retData != False:
|
if retData != False:
|
||||||
events.event('insertBlock', onionr = None, threaded = False)
|
events.event('insertBlock', onionr = None, threaded = False)
|
||||||
|
|
|
@ -199,7 +199,7 @@ class Onionr:
|
||||||
'connect': self.addAddress,
|
'connect': self.addAddress,
|
||||||
'kex': self.doKEX,
|
'kex': self.doKEX,
|
||||||
|
|
||||||
'getpassword': self.getWebPassword
|
'getpassword': self.printWebPassword
|
||||||
}
|
}
|
||||||
|
|
||||||
self.cmdhelp = {
|
self.cmdhelp = {
|
||||||
|
@ -258,6 +258,9 @@ class Onionr:
|
||||||
|
|
||||||
def getWebPassword(self):
|
def getWebPassword(self):
|
||||||
return config.get('client.hmac')
|
return config.get('client.hmac')
|
||||||
|
|
||||||
|
def printWebPassword(self):
|
||||||
|
print(self.getWebPassword())
|
||||||
|
|
||||||
def getHelp(self):
|
def getHelp(self):
|
||||||
return self.cmdhelp
|
return self.cmdhelp
|
||||||
|
|
|
@ -42,6 +42,10 @@ class InvalidHexHash(Exception):
|
||||||
'''When a string is not a valid hex string of appropriate length for a hash value'''
|
'''When a string is not a valid hex string of appropriate length for a hash value'''
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
class InvalidProof(Exception):
|
||||||
|
'''When a proof is invalid or inadequate'''
|
||||||
|
pass
|
||||||
|
|
||||||
# network level exceptions
|
# network level exceptions
|
||||||
class MissingPort(Exception):
|
class MissingPort(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
|
@ -54,21 +54,12 @@ class OnionrUtils:
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
logger.error('Failed to fetch time bypass token.', error=error)
|
logger.error('Failed to fetch time bypass token.', error=error)
|
||||||
|
|
||||||
def sendPM(self, pubkey, message):
|
def getRoundedEpoch(self, roundS=60):
|
||||||
'''
|
'''
|
||||||
High level function to encrypt a message to a peer and insert it as a block
|
Returns the epoch, rounded down to given seconds (Default 60)
|
||||||
'''
|
|
||||||
|
|
||||||
self._core.insertBlock(message, header='pm', sign=True, encryptType='asym', asymPeer=pubkey)
|
|
||||||
|
|
||||||
return
|
|
||||||
|
|
||||||
def getCurrentHourEpoch(self):
|
|
||||||
'''
|
|
||||||
Returns the current epoch, rounded down to the hour
|
|
||||||
'''
|
'''
|
||||||
epoch = self.getEpoch()
|
epoch = self.getEpoch()
|
||||||
return epoch - (epoch % 3600)
|
return epoch - (epoch % roundS)
|
||||||
|
|
||||||
def incrementAddressSuccess(self, address):
|
def incrementAddressSuccess(self, address):
|
||||||
'''
|
'''
|
||||||
|
@ -134,9 +125,10 @@ class OnionrUtils:
|
||||||
if newAdderList != False:
|
if newAdderList != False:
|
||||||
for adder in newAdderList.split(','):
|
for adder in newAdderList.split(','):
|
||||||
if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress():
|
if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress():
|
||||||
if self._core.addAddress(adder):
|
if adder[:4] == '0000':
|
||||||
logger.info('Added %s to db.' % adder, timestamp = True)
|
if self._core.addAddress(adder):
|
||||||
retVal = True
|
logger.info('Added %s to db.' % adder, timestamp = True)
|
||||||
|
retVal = True
|
||||||
else:
|
else:
|
||||||
pass
|
pass
|
||||||
#logger.debug('%s is either our address or already in our DB' % adder)
|
#logger.debug('%s is either our address or already in our DB' % adder)
|
||||||
|
@ -210,19 +202,26 @@ class OnionrUtils:
|
||||||
|
|
||||||
'''
|
'''
|
||||||
meta = {}
|
meta = {}
|
||||||
|
metadata = {}
|
||||||
|
data = blockData
|
||||||
try:
|
try:
|
||||||
blockData = blockData.encode()
|
blockData = blockData.encode()
|
||||||
except AttributeError:
|
except AttributeError:
|
||||||
pass
|
pass
|
||||||
metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
|
|
||||||
data = blockData[blockData.find(b'\n'):].decode()
|
try:
|
||||||
|
metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
|
||||||
|
except json.decoder.JSONDecodeError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
data = blockData[blockData.find(b'\n'):].decode()
|
||||||
|
|
||||||
if not metadata['encryptType'] in ('asym', 'sym'):
|
if not metadata['encryptType'] in ('asym', 'sym'):
|
||||||
try:
|
try:
|
||||||
meta = json.loads(metadata['meta'])
|
meta = json.loads(metadata['meta'])
|
||||||
except KeyError:
|
except KeyError:
|
||||||
pass
|
pass
|
||||||
meta = metadata['meta']
|
meta = metadata['meta']
|
||||||
return (metadata, meta, data)
|
return (metadata, meta, data)
|
||||||
|
|
||||||
def checkPort(self, port, host=''):
|
def checkPort(self, port, host=''):
|
||||||
|
@ -525,6 +524,30 @@ class OnionrUtils:
|
||||||
'''returns epoch'''
|
'''returns epoch'''
|
||||||
return math.floor(time.time())
|
return math.floor(time.time())
|
||||||
|
|
||||||
|
def doPostRequest(self, url, data={}, port=0, proxyType='tor'):
|
||||||
|
'''
|
||||||
|
Do a POST request through a local tor or i2p instance
|
||||||
|
'''
|
||||||
|
if proxyType == 'tor':
|
||||||
|
if port == 0:
|
||||||
|
port = self._core.torPort
|
||||||
|
proxies = {'http': 'socks5://127.0.0.1:' + str(port), 'https': 'socks5://127.0.0.1:' + str(port)}
|
||||||
|
elif proxyType == 'i2p':
|
||||||
|
proxies = {'http': 'http://127.0.0.1:4444'}
|
||||||
|
else:
|
||||||
|
return
|
||||||
|
headers = {'user-agent': 'PyOnionr'}
|
||||||
|
try:
|
||||||
|
proxies = {'http': 'socks5h://127.0.0.1:' + str(port), 'https': 'socks5h://127.0.0.1:' + str(port)}
|
||||||
|
r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
|
||||||
|
retData = r.text
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
raise KeyboardInterrupt
|
||||||
|
except requests.exceptions.RequestException as e:
|
||||||
|
logger.debug('Error: %s' % str(e))
|
||||||
|
retData = False
|
||||||
|
return retData
|
||||||
|
|
||||||
def doGetRequest(self, url, port=0, proxyType='tor'):
|
def doGetRequest(self, url, port=0, proxyType='tor'):
|
||||||
'''
|
'''
|
||||||
Do a get request through a local tor or i2p instance
|
Do a get request through a local tor or i2p instance
|
||||||
|
@ -549,7 +572,7 @@ class OnionrUtils:
|
||||||
retData = False
|
retData = False
|
||||||
return retData
|
return retData
|
||||||
|
|
||||||
def getNistBeaconSalt(self, torPort=0):
|
def getNistBeaconSalt(self, torPort=0, rounding=3600):
|
||||||
'''
|
'''
|
||||||
Get the token for the current hour from the NIST randomness beacon
|
Get the token for the current hour from the NIST randomness beacon
|
||||||
'''
|
'''
|
||||||
|
@ -559,7 +582,7 @@ class OnionrUtils:
|
||||||
except IndexError:
|
except IndexError:
|
||||||
raise onionrexceptions.MissingPort('Missing Tor socks port')
|
raise onionrexceptions.MissingPort('Missing Tor socks port')
|
||||||
retData = ''
|
retData = ''
|
||||||
curTime = self._core._utils.getCurrentHourEpoch
|
curTime = self.getRoundedEpoch(rounding)
|
||||||
self.nistSaltTimestamp = curTime
|
self.nistSaltTimestamp = curTime
|
||||||
data = self.doGetRequest('https://beacon.nist.gov/rest/record/' + str(curTime), port=torPort)
|
data = self.doGetRequest('https://beacon.nist.gov/rest/record/' + str(curTime), port=torPort)
|
||||||
dataXML = minidom.parseString(data, forbid_dtd=True, forbid_entities=True, forbid_external=True)
|
dataXML = minidom.parseString(data, forbid_dtd=True, forbid_entities=True, forbid_external=True)
|
||||||
|
|
|
@ -33,7 +33,7 @@
|
||||||
},
|
},
|
||||||
|
|
||||||
"allocations":{
|
"allocations":{
|
||||||
"disk": 1000000000,
|
"disk": 9000000000,
|
||||||
"netTotal": 1000000000,
|
"netTotal": 1000000000,
|
||||||
"blockCache" : 5000000,
|
"blockCache" : 5000000,
|
||||||
"blockCacheTotal" : 50000000
|
"blockCacheTotal" : 50000000
|
||||||
|
|
Loading…
Reference in New Issue