+ Reformatted API, more efficient, standard, and secure now

* Various bug fixes
master
Kevin Froman 2018-12-22 13:02:09 -06:00
parent 53f98c3449
commit 1dd471b91e
No known key found for this signature in database
GPG Key ID: 0D414D0FE405B63B
11 changed files with 174 additions and 102 deletions

View File

@ -25,8 +25,6 @@ import core
from onionrblockapi import Block from onionrblockapi import Block
import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionr import onionrutils, onionrexceptions, onionrcrypto, blockimporter, onionrevents as events, logger, config, onionr
API_VERSION = 0
def guessMime(path): def guessMime(path):
''' '''
Guesses the mime type of a file from the input filename Guesses the mime type of a file from the input filename
@ -67,6 +65,7 @@ class PublicAPI:
self.torAdder = clientAPI._core.hsAddress self.torAdder = clientAPI._core.hsAddress
self.i2pAdder = clientAPI._core.i2pAddress self.i2pAdder = clientAPI._core.i2pAddress
self.bindPort = config.get('client.public.port') self.bindPort = config.get('client.public.port')
logger.info('Running public api on %s:%s' % (self.host, self.bindPort))
@app.before_request @app.before_request
def validateRequest(): def validateRequest():
@ -84,7 +83,7 @@ class PublicAPI:
resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
resp.headers['X-Frame-Options'] = 'deny' resp.headers['X-Frame-Options'] = 'deny'
resp.headers['X-Content-Type-Options'] = "nosniff" resp.headers['X-Content-Type-Options'] = "nosniff"
resp.headers['X-API'] = API_VERSION resp.headers['X-API'] = onionr.API_VERSION
return resp return resp
@app.route('/') @app.route('/')
@ -105,8 +104,9 @@ class PublicAPI:
return Response('\n'.join(bList)) return Response('\n'.join(bList))
@app.route('/getdata/<name>') @app.route('/getdata/<name>')
def getBlockData(): def getBlockData(name):
resp = '' resp = ''
data = name
if clientAPI._utils.validateHash(data): if clientAPI._utils.validateHash(data):
if data not in self.hideBlocks: if data not in self.hideBlocks:
if os.path.exists(clientAPI._core.dataDir + 'blocks/' + data + '.dat'): if os.path.exists(clientAPI._core.dataDir + 'blocks/' + data + '.dat'):
@ -117,20 +117,64 @@ class PublicAPI:
resp = "" resp = ""
return Response(resp) return Response(resp)
@app.route('/www/<path:path>')
def wwwPublic(path):
if not config.get("www.public.run", True):
abort(403)
return send_from_directory(config.get('www.public.path', 'static-data/www/public/'), path)
@app.route('/ping') @app.route('/ping')
def ping(): def ping():
return Response("pong!") return Response("pong!")
@app.route('/getdbhash') @app.route('/getdbhash')
def getDBHash(): def getDBHash():
return Response(clientAPI._utils.getBlockDBHash()) return Response(clientAPI._utils.getBlockDBHash())
@app.route('/pex') @app.route('/pex')
def peerExchange(): def peerExchange():
response = ','.join(self._core.listAdders()) response = ','.join(clientAPI._core.listAdders())
if len(response) == 0: if len(response) == 0:
response = 'none' response = 'none'
resp = Response(response) return Response(response)
@app.route('/announce', methods=['post'])
def acceptAnnounce():
resp = 'failure'
powHash = ''
randomData = ''
newNode = ''
ourAdder = clientAPI._core.hsAddress.encode()
try:
newNode = request.form['node'].encode()
except KeyError:
logger.warn('No block specified for upload')
pass
else:
try:
randomData = request.form['random']
randomData = base64.b64decode(randomData)
except KeyError:
logger.warn('No random data specified for upload')
else:
nodes = newNode + clientAPI._core.hsAddress.encode()
nodes = clientAPI._core._crypto.blake2bHash(nodes)
powHash = clientAPI._core._crypto.blake2bHash(randomData + nodes)
try:
powHash = powHash.decode()
except AttributeError:
pass
if powHash.startswith('0000'):
try:
newNode = newNode.decode()
except AttributeError:
pass
if clientAPI._core.addAddress(newNode):
resp = 'Success'
else:
logger.warn(newNode.decode() + ' failed to meet POW: ' + powHash)
resp = Response(resp)
return resp
@app.route('/upload', methods=['post']) @app.route('/upload', methods=['post'])
def upload(): def upload():
@ -156,6 +200,10 @@ class PublicAPI:
return resp return resp
clientAPI.setPublicAPIInstance(self) clientAPI.setPublicAPIInstance(self)
while self.torAdder == '':
clientAPI._core.refreshFirstStartVars()
self.torAdder = clientAPI._core.hsAddress
time.sleep(1)
self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None) self.httpServer = WSGIServer((self.host, self.bindPort), app, log=None)
self.httpServer.serve_forever() self.httpServer.serve_forever()
@ -166,7 +214,7 @@ class API:
callbacks = {'public' : {}, 'private' : {}} callbacks = {'public' : {}, 'private' : {}}
def __init__(self, debug, API_VERSION): def __init__(self, onionrInst, debug, API_VERSION):
''' '''
Initialize the api server, preping variables for later use Initialize the api server, preping variables for later use
@ -190,10 +238,11 @@ class API:
self.timeBypassToken = base64.b16encode(os.urandom(32)).decode() self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
self.publicAPI = None # gets set when the thread calls our setter... bad hack but kinda necessary with flask self.publicAPI = None # gets set when the thread calls our setter... bad hack but kinda necessary with flask
threading.Thread(target=PublicAPI, args=(self,)).start() #threading.Thread(target=PublicAPI, args=(self,)).start()
self.host = setBindIP(self._core.privateApiHostFile) self.host = setBindIP(self._core.privateApiHostFile)
logger.info('Running api on %s:%s' % (self.host, self.bindPort)) logger.info('Running api on %s:%s' % (self.host, self.bindPort))
self.httpServer = '' self.httpServer = ''
onionrInst.setClientAPIInst(self)
@app.before_request @app.before_request
def validateRequest(): def validateRequest():
@ -205,20 +254,20 @@ class API:
abort(403) abort(403)
except KeyError: except KeyError:
abort(403) abort(403)
@app.after_request @app.after_request
def afterReq(resp): def afterReq(resp):
resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'" resp.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'none'; object-src 'none'; style-src data: 'unsafe-inline'; img-src data:; media-src 'none'; frame-src 'none'; font-src 'none'; connect-src 'none'"
resp.headers['X-Frame-Options'] = 'deny' resp.headers['X-Frame-Options'] = 'deny'
resp.headers['X-Content-Type-Options'] = "nosniff" resp.headers['X-Content-Type-Options'] = "nosniff"
resp.headers['X-API'] = API_VERSION resp.headers['X-API'] = onionr.API_VERSION
resp.headers['Server'] = 'nginx' resp.headers['Server'] = ''
resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch. resp.headers['Date'] = 'Thu, 1 Jan 1970 00:00:00 GMT' # Clock info is probably useful to attackers. Set to unix epoch.
return resp return resp
@app.route('/ping') @app.route('/ping')
def ping(): def ping():
return Respose("pong!") return Response("pong!")
@app.route('/') @app.route('/')
def hello(): def hello():
@ -256,10 +305,10 @@ class API:
except AttributeError: except AttributeError:
pass pass
return Response("bye") return Response("bye")
self.httpServer = WSGIServer((self.host, bindPort), app, log=None) self.httpServer = WSGIServer((self.host, bindPort), app, log=None)
self.httpServer.serve_forever() self.httpServer.serve_forever()
def setPublicAPIInstance(self, inst): def setPublicAPIInstance(self, inst):
assert isinstance(inst, PublicAPI) assert isinstance(inst, PublicAPI)
self.publicAPI = inst self.publicAPI = inst
@ -277,4 +326,4 @@ class API:
else: else:
return True return True
except TypeError: except TypeError:
return False return False

View File

@ -19,11 +19,10 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
#import gevent.monkey
#gevent.monkey.patch_all()
import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid import sys, os, core, config, json, requests, time, logger, threading, base64, onionr, uuid
import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block import onionrexceptions, onionrpeers, onionrevents as events, onionrplugins as plugins, onionrblockapi as block
import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs import onionrdaemontools, onionrsockets, onionrchat, onionr, onionrproofs
import binascii
from dependencies import secrets from dependencies import secrets
from defusedxml import minidom from defusedxml import minidom
@ -103,6 +102,7 @@ class OnionrCommunicatorDaemon:
OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True) OnionrCommunicatorTimers(self, self.daemonTools.cooldownPeer, 30, requiresPeer=True)
OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1) OnionrCommunicatorTimers(self, self.uploadBlock, 10, requiresPeer=True, maxThreads=1)
OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1) OnionrCommunicatorTimers(self, self.daemonCommands, 6, maxThreads=1)
OnionrCommunicatorTimers(self, self.detectAPICrash, 5, maxThreads=1)
deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1) deniableBlockTimer = OnionrCommunicatorTimers(self, self.daemonTools.insertDeniableBlock, 180, requiresPeer=True, maxThreads=1)
netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600) netCheckTimer = OnionrCommunicatorTimers(self, self.daemonTools.netCheck, 600)
@ -232,7 +232,10 @@ class OnionrCommunicatorDaemon:
content = content.encode() content = content.encode()
except AttributeError: except AttributeError:
pass pass
content = base64.b64decode(content) # content is base64 encoded in transport try:
content = base64.b64decode(content) # content is base64 encoded in transport
except binascii.Error:
pass
realHash = self._core._crypto.sha3Hash(content) realHash = self._core._crypto.sha3Hash(content)
try: try:
realHash = realHash.decode() # bytes on some versions for some reason realHash = realHash.decode() # bytes on some versions for some reason
@ -423,7 +426,7 @@ class OnionrCommunicatorDaemon:
if len(peer) == 0: if len(peer) == 0:
return False return False
#logger.debug('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort)) #logger.debug('Performing ' + action + ' with ' + peer + ' on port ' + str(self.proxyPort))
url = 'http://' + peer + '/' + action url = 'http://%s/%s' % (peer, action)
if len(data) > 0: if len(data) > 0:
url += '&data=' + data url += '&data=' + data
@ -546,9 +549,9 @@ class OnionrCommunicatorDaemon:
def detectAPICrash(self): def detectAPICrash(self):
'''exit if the api server crashes/stops''' '''exit if the api server crashes/stops'''
if self._core._utils.localCommand('ping', silent=False) != 'pong': if self._core._utils.localCommand('ping', silent=False) not in ('pong', 'pong!'):
for i in range(5): for i in range(5):
if self._core._utils.localCommand('ping') == 'pong': if self._core._utils.localCommand('ping') in ('pong', 'pong!'):
break # break for loop break # break for loop
time.sleep(1) time.sleep(1)
else: else:

View File

@ -154,7 +154,7 @@ class Core:
if address == config.get('i2p.ownAddr', None) or address == self.hsAddress: if address == config.get('i2p.ownAddr', None) or address == self.hsAddress:
return False return False
if type(address) is type(None) or len(address) == 0: if type(address) is None or len(address) == 0:
return False return False
if self._utils.validateID(address): if self._utils.validateID(address):
conn = sqlite3.connect(self.addressDB, timeout=10) conn = sqlite3.connect(self.addressDB, timeout=10)

View File

@ -77,7 +77,7 @@ class NetController:
hsVer = '# v2 onions' hsVer = '# v2 onions'
if config.get('tor.v3onions'): if config.get('tor.v3onions'):
hsVer = 'HiddenServiceVersion 3' hsVer = 'HiddenServiceVersion 3'
logger.debug('Using v3 onions :)') logger.debug('Using v3 onions')
if os.path.exists(self.torConfigLocation): if os.path.exists(self.torConfigLocation):
os.remove(self.torConfigLocation) os.remove(self.torConfigLocation)

View File

@ -67,6 +67,7 @@ class Onionr:
data_exists = Onionr.setupConfig(self.dataDir, self = self) data_exists = Onionr.setupConfig(self.dataDir, self = self)
self.onionrCore = core.Core() self.onionrCore = core.Core()
#self.deleteRunFiles()
self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore) self.onionrUtils = onionrutils.OnionrUtils(self.onionrCore)
self.clientAPIInst = '' # Client http api instance self.clientAPIInst = '' # Client http api instance
@ -399,6 +400,16 @@ class Onionr:
logger.info('Syntax: friend add/remove/list [address]') logger.info('Syntax: friend add/remove/list [address]')
def deleteRunFiles(self):
try:
os.remove(self.onionrCore.publicApiHostFile)
except FileNotFoundError:
pass
try:
os.remove(self.onionrCore.privateApiHostFile)
except FileNotFoundError:
pass
def banBlock(self): def banBlock(self):
try: try:
ban = sys.argv[2] ban = sys.argv[2]
@ -695,6 +706,13 @@ class Onionr:
os.remove('.onionr-lock') os.remove('.onionr-lock')
except FileNotFoundError: except FileNotFoundError:
pass pass
def setClientAPIInst(self, inst):
self.clientAPIInst = inst
def getClientApi(self):
while self.clientAPIInst == '':
time.sleep(0.5)
return self.clientAPIInst
def daemon(self): def daemon(self):
''' '''
@ -708,64 +726,66 @@ class Onionr:
logger.debug('Runcheck file found on daemon start, deleting in advance.') logger.debug('Runcheck file found on daemon start, deleting in advance.')
os.remove('data/.runcheck') os.remove('data/.runcheck')
apiTarget = api.API Thread(target=api.API, args=(self, self.debug, API_VERSION)).start()
apiThread = Thread(target = apiTarget, args = (self.debug, API_VERSION)) Thread(target=api.PublicAPI, args=[self.getClientApi()]).start()
apiThread.start()
try: try:
time.sleep(3) time.sleep(0)
except KeyboardInterrupt: except KeyboardInterrupt:
logger.debug('Got keyboard interrupt, shutting down...') logger.debug('Got keyboard interrupt, shutting down...')
time.sleep(1) time.sleep(1)
self.onionrUtils.localCommand('shutdown') self.onionrUtils.localCommand('shutdown')
apiHost = ''
while apiHost == '':
try:
with open(self.onionrCore.publicApiHostFile, 'r') as hostFile:
apiHost = hostFile.read()
except FileNotFoundError:
pass
time.sleep(0.5)
Onionr.setupConfig('data/', self = self)
if self._developmentMode:
logger.warn('DEVELOPMENT MODE ENABLED (LESS SECURE)', timestamp = False)
net = NetController(config.get('client.public.port', 59497), apiServerIP=apiHost)
logger.debug('Tor is starting...')
if not net.startTor():
self.onionrUtils.localCommand('shutdown')
sys.exit(1)
if len(net.myID) > 0 and config.get('general.security_level') == 0:
logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID))
else: else:
apiHost = '127.0.0.1' logger.debug('.onion service disabled')
if apiThread.isAlive(): logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey))
try: time.sleep(1)
with open(self.onionrCore.publicApiHostFile, 'r') as hostFile:
apiHost = hostFile.read()
except FileNotFoundError:
pass
Onionr.setupConfig('data/', self = self)
if self._developmentMode: # TODO: make runable on windows
logger.warn('DEVELOPMENT MODE ENABLED (LESS SECURE)', timestamp = False) communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)])
net = NetController(config.get('client.public.port', 59497), apiServerIP=apiHost)
logger.debug('Tor is starting...')
if not net.startTor():
self.onionrUtils.localCommand('shutdown')
sys.exit(1)
if len(net.myID) > 0 and config.get('general.security_level') == 0:
logger.debug('Started .onion service: %s' % (logger.colors.underline + net.myID))
else:
logger.debug('.onion service disabled')
logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey))
time.sleep(1)
# TODO: make runable on windows # print nice header thing :)
communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)]) if config.get('general.display_header', True):
self.header()
# print nice header thing :) # print out debug info
if config.get('general.display_header', True): self.version(verbosity = 5, function = logger.debug)
self.header() logger.debug('Python version %s' % platform.python_version())
# print out debug info logger.debug('Started communicator.')
self.version(verbosity = 5, function = logger.debug)
logger.debug('Python version %s' % platform.python_version())
logger.debug('Started communicator.') events.event('daemon_start', onionr = self)
try:
while True:
time.sleep(5)
events.event('daemon_start', onionr = self) # Break if communicator process ends, so we don't have left over processes
try: if communicatorProc.poll() is not None:
while True: break
time.sleep(5) except KeyboardInterrupt:
self.onionrCore.daemonQueueAdd('shutdown')
# Break if communicator process ends, so we don't have left over processes self.onionrUtils.localCommand('shutdown')
if communicatorProc.poll() is not None: time.sleep(3)
break self.deleteRunFiles()
except KeyboardInterrupt: net.killTor()
self.onionrCore.daemonQueueAdd('shutdown')
self.onionrUtils.localCommand('shutdown')
return return
def killDaemon(self): def killDaemon(self):

View File

@ -45,7 +45,7 @@ class DaemonTools:
ourID = self.daemon._core.hsAddress.strip() ourID = self.daemon._core.hsAddress.strip()
url = 'http://' + peer + '/public/announce/' url = 'http://' + peer + '/announce'
data = {'node': ourID} data = {'node': ourID}
combinedNodes = ourID + peer combinedNodes = ourID + peer

View File

@ -79,27 +79,29 @@ def peerCleanup(coreInst):
logger.info('Cleaning peers...') logger.info('Cleaning peers...')
config.reload() config.reload()
minScore = int(config.get('peers.minimum_score', -100))
maxPeers = int(config.get('peers.max_stored', 5000))
adders = getScoreSortedPeerList(coreInst) adders = getScoreSortedPeerList(coreInst)
adders.reverse() adders.reverse()
if len(adders) > 1:
for address in adders: minScore = int(config.get('peers.minimum_score', -100))
# Remove peers that go below the negative score maxPeers = int(config.get('peers.max_stored', 5000))
if PeerProfiles(address, coreInst).score < minScore:
coreInst.removeAddress(address) for address in adders:
try: # Remove peers that go below the negative score
if (int(coreInst._utils.getEpoch()) - int(coreInst.getPeerInfo(address, 'dateSeen'))) >= 600: if PeerProfiles(address, coreInst).score < minScore:
expireTime = 600 coreInst.removeAddress(address)
else: try:
expireTime = 86400 if (int(coreInst._utils.getEpoch()) - int(coreInst.getPeerInfo(address, 'dateSeen'))) >= 600:
coreInst._blacklist.addToDB(address, dataType=1, expire=expireTime) expireTime = 600
except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue else:
pass expireTime = 86400
except ValueError: coreInst._blacklist.addToDB(address, dataType=1, expire=expireTime)
pass except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue
logger.warn('Removed address ' + address + '.') pass
except ValueError:
pass
logger.warn('Removed address ' + address + '.')
# Unban probably not malicious peers TODO improve # Unban probably not malicious peers TODO improve
coreInst._blacklist.deleteExpired(dataType=1) coreInst._blacklist.deleteExpired(dataType=1)

View File

@ -238,7 +238,6 @@ class POW:
break break
else: else:
time.sleep(2) time.sleep(2)
print('boi')
except KeyboardInterrupt: except KeyboardInterrupt:
self.shutdown() self.shutdown()
logger.warn('Got keyboard interrupt while waiting for POW result, stopping') logger.warn('Got keyboard interrupt while waiting for POW result, stopping')

View File

@ -159,17 +159,17 @@ class OnionrUtils:
config.reload() config.reload()
self.getTimeBypassToken() self.getTimeBypassToken()
# TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless. # TODO: URL encode parameters, just as an extra measure. May not be needed, but should be added regardless.
try: hostname = ''
with open(self._core.privateApiHostFile, 'r') as host: while hostname == '':
hostname = host.read() try:
except FileNotFoundError: with open(self._core.privateApiHostFile, 'r') as host:
return False hostname = host.read()
except FileNotFoundError:
print('wat')
time.sleep(1)
if data != '': if data != '':
data = '&data=' + urllib.parse.quote_plus(data) data = '&data=' + urllib.parse.quote_plus(data)
payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data) payload = 'http://%s:%s/%s%s' % (hostname, config.get('client.client.port'), command, data)
#payload = 'http://%s:%s/client/?action=%s&token=%s&timingToken=%s' % (hostname, config.get('client.client.port'), command, config.get('client.webpassword'), self.timingToken)
#if data != '':
# payload += '&data=' + urllib.parse.quote_plus(data)
try: try:
retData = requests.get(payload, headers={'token': config.get('client.webpassword')}).text retData = requests.get(payload, headers={'token': config.get('client.webpassword')}).text
except Exception as error: except Exception as error:

View File

@ -1,7 +1,7 @@
<h1>This is an Onionr Node</h1> <h1>This is an Onionr Node</h1>
<p>The content on this server is not necessarily created by the server owner, and was not necessarily stored specifically with the owner's knowledge of its contents.</p> <p>The content on this server was not necessarily intentionally stored or created by the owner(s) of this server.</p>
<p>Onionr is a decentralized data storage system that anyone can insert data into.</p> <p>Onionr is a decentralized peer-to-peer data storage system.</p>
<p>To learn more about Onionr, see the website at <a href="https://onionr.voidnet.tech/">https://Onionr.VoidNet.tech/</a></p> <p>To learn more about Onionr, see the website at <a href="https://onionr.voidnet.tech/">https://Onionr.VoidNet.tech/</a></p>

View File

@ -2,7 +2,6 @@ urllib3==1.23
requests==2.20.0 requests==2.20.0
PyNaCl==1.2.1 PyNaCl==1.2.1
gevent==1.3.6 gevent==1.3.6
sha3==0.2.1
defusedxml==0.5.0 defusedxml==0.5.0
Flask==1.0.2 Flask==1.0.2
PySocks==1.6.8 PySocks==1.6.8