Merge branch 'node-pow' into 'master'

Reverse block insertion and other bug fixes

See merge request beardog/Onionr!4
master
Kevin 2018-07-29 04:51:48 +00:00
commit 83c4dbcb72
9 changed files with 156 additions and 60 deletions

View File

@ -24,7 +24,7 @@ from gevent.wsgi import WSGIServer
import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config
from core import Core from core import Core
from onionrblockapi import Block from onionrblockapi import Block
import onionrutils, onionrcrypto import onionrutils, onionrcrypto, blockimporter
class API: class API:
''' '''
@ -141,9 +141,6 @@ class API:
resp = Response('Goodbye') resp = Response('Goodbye')
elif action == 'ping': elif action == 'ping':
resp = Response('pong') resp = Response('pong')
elif action == 'stats':
resp = Response('me_irl')
raise Exception
elif action == 'site': elif action == 'site':
block = data block = data
siteData = self._core.getData(data) siteData = self._core.getData(data)
@ -175,6 +172,24 @@ class API:
resp = Response("") resp = Response("")
return resp return resp
@app.route('/public/upload/', methods=['POST'])
def blockUpload():
self.validateHost('public')
resp = 'failure'
try:
data = request.form['block']
except KeyError:
logger.warn('No block specified for upload')
pass
else:
if sys.getsizeof(data) < 100000000:
if blockimporter.importBlockFromData(data, self._core):
resp = 'success'
else:
logger.warn('Error encountered importing uploaded block')
resp = Response(resp)
return resp
@app.route('/public/') @app.route('/public/')
def public_handler(): def public_handler():
# Public means it is publicly network accessible # Public means it is publicly network accessible
@ -198,6 +213,7 @@ class API:
resp = Response('\n'.join(self._core.getBlockList())) resp = Response('\n'.join(self._core.getBlockList()))
elif action == 'directMessage': elif action == 'directMessage':
resp = Response(self._core.handle_direct_connection(data)) resp = Response(self._core.handle_direct_connection(data))
elif action == 'announce': elif action == 'announce':
if data != '': if data != '':
# TODO: require POW for this # TODO: require POW for this

40
onionr/blockimporter.py Normal file
View File

@ -0,0 +1,40 @@
'''
Onionr - P2P Microblogging Platform & Social network
Import block data and save it
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import core, onionrexceptions, logger
def importBlockFromData(content, coreInst):
retData = False
if not isinstance(coreInst, core.Core):
raise Exception("coreInst must be an Onionr core instance")
try:
content = content.encode()
except AttributeError:
pass
metas = coreInst._utils.getBlockMetadataFromData(content) # returns tuple(metadata, meta), meta is also in metadata
metadata = metas[0]
if coreInst._utils.validateMetadata(metadata): # check if metadata is valid
if coreInst._crypto.verifyPow(content): # check if POW is enough/correct
logger.info('Block passed proof, saving.')
blockHash = coreInst.setData(content)
blockHash = coreInst.addToBlockDB(blockHash, dataSaved=True)
coreInst._utils.processBlockMetadata(blockHash) # caches block metadata values to block database
retData = True
return retData

View File

@ -36,6 +36,8 @@ class OnionrCommunicatorDaemon:
# intalize NIST beacon salt and time # intalize NIST beacon salt and time
self.nistSaltTimestamp = 0 self.nistSaltTimestamp = 0
self.powSalt = 0 self.powSalt = 0
self.blockToUpload = ''
# loop time.sleep delay in seconds # loop time.sleep delay in seconds
self.delay = 1 self.delay = 1
@ -84,7 +86,7 @@ class OnionrCommunicatorDaemon:
OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True) OnionrCommunicatorTimers(self, self.lookupAdders, 60, requiresPeer=True)
# set loop to execute instantly to load up peer pool (replaced old pool init wait) # set loop to execute instantly to load up peer pool (replaced old pool init wait)
peerPoolTimer.count = (peerPoolTimer.frequency - 1) peerPoolTimer.count = (peerPoolTimer.frequency - 1)
# Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking # Main daemon loop, mainly for calling timers, don't do any complex operations here to avoid locking
try: try:
@ -101,7 +103,7 @@ class OnionrCommunicatorDaemon:
logger.info('Goodbye.') logger.info('Goodbye.')
self._core._utils.localCommand('shutdown') self._core._utils.localCommand('shutdown')
time.sleep(0.5) time.sleep(0.5)
def lookupKeys(self): def lookupKeys(self):
'''Lookup new keys''' '''Lookup new keys'''
logger.debug('Looking up new keys...') logger.debug('Looking up new keys...')
@ -111,7 +113,6 @@ class OnionrCommunicatorDaemon:
peer = self.pickOnlinePeer() peer = self.pickOnlinePeer()
newKeys = self.peerAction(peer, action='kex') newKeys = self.peerAction(peer, action='kex')
self._core._utils.mergeKeys(newKeys) self._core._utils.mergeKeys(newKeys)
self.decrementThreadCount('lookupKeys') self.decrementThreadCount('lookupKeys')
return return
@ -196,7 +197,7 @@ class OnionrCommunicatorDaemon:
pass pass
logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash) logger.warn('Block hash validation failed for ' + blockHash + ' got ' + tempHash)
self.blockQueue.remove(blockHash) # remove from block queue both if success or false self.blockQueue.remove(blockHash) # remove from block queue both if success or false
self.currentDownloading.remove(blockHash) self.currentDownloading.remove(blockHash)
self.decrementThreadCount('getBlocks') self.decrementThreadCount('getBlocks')
return return
@ -339,10 +340,32 @@ class OnionrCommunicatorDaemon:
for i in self.timers: for i in self.timers:
if i.timerFunction.__name__ == 'lookupKeys': if i.timerFunction.__name__ == 'lookupKeys':
i.count = (i.frequency - 1) i.count = (i.frequency - 1)
elif cmd[0] == 'uploadBlock':
self.blockToUpload = cmd[1]
threading.Thread(target=self.uploadBlock).start()
else: else:
logger.info('Recieved daemonQueue command:' + cmd[0]) logger.info('Recieved daemonQueue command:' + cmd[0])
self.decrementThreadCount('daemonCommands') self.decrementThreadCount('daemonCommands')
def uploadBlock(self):
triedPeers = []
if not self._core._utils.validateHash(self.blockToUpload):
logger.warn('Requested to upload invalid block')
return
for i in range(max(len(self.onlinePeers), 2)):
peer = self.pickOnlinePeer()
if peer in triedPeers:
continue
triedPeers.append(peer)
url = 'http://' + peer + '/public/upload/'
data = {'block': block.Block(self.blockToUpload).getRaw()}
if peer.endswith('.onion'):
proxyType = 'tor'
elif peer.endswith('.i2p'):
proxyType = 'i2p'
logger.info("Uploading block")
self._core._utils.doPostRequest(url, data=data, proxyType=proxyType)
def announce(self, peer): def announce(self, peer):
'''Announce to peers our address''' '''Announce to peers our address'''
announceCount = 0 announceCount = 0

View File

@ -41,10 +41,12 @@ class Core:
self.blockDataLocation = 'data/blocks/' self.blockDataLocation = 'data/blocks/'
self.addressDB = 'data/address.db' self.addressDB = 'data/address.db'
self.hsAdder = '' self.hsAdder = ''
self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt' self.bootstrapFileLocation = 'static-data/bootstrap-nodes.txt'
self.bootstrapList = [] self.bootstrapList = []
self.requirements = onionrvalues.OnionrValues() self.requirements = onionrvalues.OnionrValues()
self.torPort = torPort
self.usageFile = 'data/disk-usage.txt'
if not os.path.exists('data/'): if not os.path.exists('data/'):
os.mkdir('data/') os.mkdir('data/')
@ -578,25 +580,6 @@ class Core:
conn.close() conn.close()
return return
def handle_direct_connection(self, data):
'''
Handles direct messages
'''
try:
data = json.loads(data)
# TODO: Determine the sender, verify, etc
if ('callback' in data) and (data['callback'] is True):
# then this is a response to the message we sent earlier
self.daemonQueueAdd('checkCallbacks', json.dumps(data))
else:
# then we should handle it and respond accordingly
self.daemonQueueAdd('incomingDirectConnection', json.dumps(data))
except Exception as e:
logger.warn('Failed to handle incoming direct message: %s' % str(e))
return
def getBlockList(self, unsaved = False): # TODO: Use unsaved?? def getBlockList(self, unsaved = False): # TODO: Use unsaved??
''' '''
Get list of our blocks Get list of our blocks
@ -757,6 +740,7 @@ class Core:
retData = self.setData(payload) retData = self.setData(payload)
self.addToBlockDB(retData, selfInsert=True, dataSaved=True) self.addToBlockDB(retData, selfInsert=True, dataSaved=True)
self.setBlockType(retData, meta['type']) self.setBlockType(retData, meta['type'])
self.daemonQueueAdd('uploadBlock', retData)
if retData != False: if retData != False:
events.event('insertBlock', onionr = None, threaded = False) events.event('insertBlock', onionr = None, threaded = False)

View File

@ -199,7 +199,7 @@ class Onionr:
'connect': self.addAddress, 'connect': self.addAddress,
'kex': self.doKEX, 'kex': self.doKEX,
'getpassword': self.getWebPassword 'getpassword': self.printWebPassword
} }
self.cmdhelp = { self.cmdhelp = {
@ -258,6 +258,9 @@ class Onionr:
def getWebPassword(self): def getWebPassword(self):
return config.get('client.hmac') return config.get('client.hmac')
def printWebPassword(self):
print(self.getWebPassword())
def getHelp(self): def getHelp(self):
return self.cmdhelp return self.cmdhelp

View File

@ -42,6 +42,10 @@ class InvalidHexHash(Exception):
'''When a string is not a valid hex string of appropriate length for a hash value''' '''When a string is not a valid hex string of appropriate length for a hash value'''
pass pass
class InvalidProof(Exception):
'''When a proof is invalid or inadequate'''
pass
# network level exceptions # network level exceptions
class MissingPort(Exception): class MissingPort(Exception):
pass pass

View File

@ -22,16 +22,19 @@ import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, lo
import core import core
class DataPOW: class DataPOW:
def __init__(self, data, threadCount = 5): def __init__(self, data, forceDifficulty=0, threadCount = 5):
self.foundHash = False self.foundHash = False
self.difficulty = 0 self.difficulty = 0
self.data = data self.data = data
self.threadCount = threadCount self.threadCount = threadCount
dataLen = sys.getsizeof(data) if forceDifficulty == 0:
self.difficulty = math.floor(dataLen / 1000000) dataLen = sys.getsizeof(data)
if self.difficulty <= 2: self.difficulty = math.floor(dataLen / 1000000)
self.difficulty = 4 if self.difficulty <= 2:
self.difficulty = 4
else:
self.difficulty = forceDifficulty
try: try:
self.data = self.data.encode() self.data = self.data.encode()

View File

@ -54,21 +54,12 @@ class OnionrUtils:
except Exception as error: except Exception as error:
logger.error('Failed to fetch time bypass token.', error=error) logger.error('Failed to fetch time bypass token.', error=error)
def sendPM(self, pubkey, message): def getRoundedEpoch(self, roundS=60):
''' '''
High level function to encrypt a message to a peer and insert it as a block Returns the epoch, rounded down to given seconds (Default 60)
'''
self._core.insertBlock(message, header='pm', sign=True, encryptType='asym', asymPeer=pubkey)
return
def getCurrentHourEpoch(self):
'''
Returns the current epoch, rounded down to the hour
''' '''
epoch = self.getEpoch() epoch = self.getEpoch()
return epoch - (epoch % 3600) return epoch - (epoch % roundS)
def incrementAddressSuccess(self, address): def incrementAddressSuccess(self, address):
''' '''
@ -134,9 +125,10 @@ class OnionrUtils:
if newAdderList != False: if newAdderList != False:
for adder in newAdderList.split(','): for adder in newAdderList.split(','):
if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress(): if not adder in self._core.listAdders(randomOrder = False) and adder.strip() != self.getMyAddress():
if self._core.addAddress(adder): if adder[:4] == '0000':
logger.info('Added %s to db.' % adder, timestamp = True) if self._core.addAddress(adder):
retVal = True logger.info('Added %s to db.' % adder, timestamp = True)
retVal = True
else: else:
pass pass
#logger.debug('%s is either our address or already in our DB' % adder) #logger.debug('%s is either our address or already in our DB' % adder)
@ -210,19 +202,26 @@ class OnionrUtils:
''' '''
meta = {} meta = {}
metadata = {}
data = blockData
try: try:
blockData = blockData.encode() blockData = blockData.encode()
except AttributeError: except AttributeError:
pass pass
metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
data = blockData[blockData.find(b'\n'):].decode() try:
metadata = json.loads(blockData[:blockData.find(b'\n')].decode())
except json.decoder.JSONDecodeError:
pass
else:
data = blockData[blockData.find(b'\n'):].decode()
if not metadata['encryptType'] in ('asym', 'sym'): if not metadata['encryptType'] in ('asym', 'sym'):
try: try:
meta = json.loads(metadata['meta']) meta = json.loads(metadata['meta'])
except KeyError: except KeyError:
pass pass
meta = metadata['meta'] meta = metadata['meta']
return (metadata, meta, data) return (metadata, meta, data)
def checkPort(self, port, host=''): def checkPort(self, port, host=''):
@ -525,6 +524,30 @@ class OnionrUtils:
'''returns epoch''' '''returns epoch'''
return math.floor(time.time()) return math.floor(time.time())
def doPostRequest(self, url, data={}, port=0, proxyType='tor'):
'''
Do a POST request through a local tor or i2p instance
'''
if proxyType == 'tor':
if port == 0:
port = self._core.torPort
proxies = {'http': 'socks5://127.0.0.1:' + str(port), 'https': 'socks5://127.0.0.1:' + str(port)}
elif proxyType == 'i2p':
proxies = {'http': 'http://127.0.0.1:4444'}
else:
return
headers = {'user-agent': 'PyOnionr'}
try:
proxies = {'http': 'socks5h://127.0.0.1:' + str(port), 'https': 'socks5h://127.0.0.1:' + str(port)}
r = requests.post(url, data=data, headers=headers, proxies=proxies, allow_redirects=False, timeout=(15, 30))
retData = r.text
except KeyboardInterrupt:
raise KeyboardInterrupt
except requests.exceptions.RequestException as e:
logger.debug('Error: %s' % str(e))
retData = False
return retData
def doGetRequest(self, url, port=0, proxyType='tor'): def doGetRequest(self, url, port=0, proxyType='tor'):
''' '''
Do a get request through a local tor or i2p instance Do a get request through a local tor or i2p instance
@ -549,7 +572,7 @@ class OnionrUtils:
retData = False retData = False
return retData return retData
def getNistBeaconSalt(self, torPort=0): def getNistBeaconSalt(self, torPort=0, rounding=3600):
''' '''
Get the token for the current hour from the NIST randomness beacon Get the token for the current hour from the NIST randomness beacon
''' '''
@ -559,7 +582,7 @@ class OnionrUtils:
except IndexError: except IndexError:
raise onionrexceptions.MissingPort('Missing Tor socks port') raise onionrexceptions.MissingPort('Missing Tor socks port')
retData = '' retData = ''
curTime = self._core._utils.getCurrentHourEpoch curTime = self.getRoundedEpoch(rounding)
self.nistSaltTimestamp = curTime self.nistSaltTimestamp = curTime
data = self.doGetRequest('https://beacon.nist.gov/rest/record/' + str(curTime), port=torPort) data = self.doGetRequest('https://beacon.nist.gov/rest/record/' + str(curTime), port=torPort)
dataXML = minidom.parseString(data, forbid_dtd=True, forbid_entities=True, forbid_external=True) dataXML = minidom.parseString(data, forbid_dtd=True, forbid_entities=True, forbid_external=True)

View File

@ -33,7 +33,7 @@
}, },
"allocations":{ "allocations":{
"disk": 1000000000, "disk": 9000000000,
"netTotal": 1000000000, "netTotal": 1000000000,
"blockCache" : 5000000, "blockCache" : 5000000,
"blockCacheTotal" : 50000000 "blockCacheTotal" : 50000000