dxs
/
onionr
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

progress in removing core

master
Kevin Froman 2019-07-19 14:49:56 -05:00
parent 08d3e3a231
commit e12781a49d
35 changed files with 291 additions and 340 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
onionr/blockimporter.py
View File

@ -17,20 +17,20 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import core, onionrexceptions, logger import onionrexceptions, logger
from onionrutils import validatemetadata, blockmetadata from onionrutils import validatemetadata, blockmetadata
from coredb import blockmetadb from coredb import blockmetadb
def importBlockFromData(content, coreInst): import onionrcrypto, onionrblacklist, onionrstorage
def importBlockFromData(content):
crypto = onionrcrypto.OnionrCrypto()
blacklist = onionrblacklist.OnionrBlackList()
retData = False retData = False
dataHash = coreInst._crypto.sha3Hash(content) dataHash = crypto.sha3Hash(content)
if coreInst._blacklist.inBlacklist(dataHash): if blacklist.inBlacklist(dataHash):
raise onionrexceptions.BlacklistedBlock('%s is a blacklisted block' % (dataHash,)) raise onionrexceptions.BlacklistedBlock('%s is a blacklisted block' % (dataHash,))
if not isinstance(coreInst, core.Core):
raise Exception("coreInst must be an Onionr core instance")
try: try:
content = content.encode() content = content.encode()
except AttributeError: except AttributeError:
@ -38,15 +38,15 @@ def importBlockFromData(content, coreInst):
metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
metadata = metas[0] metadata = metas[0]
if validatemetadata.validate_metadata(coreInst, metadata, metas[2]): # check if metadata is valid if validatemetadata.validate_metadata(metadata, metas[2]): # check if metadata is valid
if coreInst._crypto.verifyPow(content): # check if POW is enough/correct if crypto.verifyPow(content): # check if POW is enough/correct
logger.info('Block passed proof, saving.', terminal=True) logger.info('Block passed proof, saving.', terminal=True)
try: try:
blockHash = coreInst.setData(content) blockHash = onionrstorage.setdata(content)
except onionrexceptions.DiskAllocationReached: except onionrexceptions.DiskAllocationReached:
pass pass
else: else:
blockmetadb.add_to_block_DB(blockHash, dataSaved=True) blockmetadb.add_to_block_DB(blockHash, dataSaved=True)
blockmetadata.process_block_metadata(coreInst, blockHash) # caches block metadata values to block database blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
retData = True retData = True
return retData return retData

3
onionr/coredb/dbfiles.py
View File

@ -7,4 +7,5 @@ block_data_db = '%sblocks/block-data.db' % (home,)
daemon_queue_db = '%sdaemon-queue.db' % (home,) daemon_queue_db = '%sdaemon-queue.db' % (home,)
address_info_db = '%saddress.db' % (home,) address_info_db = '%saddress.db' % (home,)
user_id_info_db = '%susers.db' % (home,) user_id_info_db = '%susers.db' % (home,)
forward_keys_db = '%sforward-keys.db' % (home,) forward_keys_db = '%sforward-keys.db' % (home,)
blacklist_db = '%sblacklist.db' % (home,)

21
onionr/keymanager.py
View File

@ -18,17 +18,15 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
from onionrutils import bytesconverter from onionrutils import bytesconverter
import onionrcrypto from onionrcrypto import generate
import filepaths
class KeyManager: class KeyManager:
def __init__(self, crypto): def __init__(self):
assert isinstance(crypto, onionrcrypto.OnionrCrypto) self.keyFile = filepaths.keys_file
self._core = crypto._core
self.keyFile = crypto._keyFile
self.crypto = crypto
def addKey(self, pubKey=None, privKey=None): def addKey(self, pubKey=None, privKey=None):
if type(pubKey) is type(None) and type(privKey) is type(None): if type(pubKey) is type(None) and type(privKey) is type(None):
pubKey, privKey = self.crypto.generatePubKey() pubKey, privKey = generate.generate_pub_key()
pubKey = bytesconverter.bytes_to_str(pubKey) pubKey = bytesconverter.bytes_to_str(pubKey)
privKey = bytesconverter.bytes_to_str(privKey) privKey = bytesconverter.bytes_to_str(privKey)
try: try:
@ -70,11 +68,4 @@ class KeyManager:
for pair in keyData.split('\n'): for pair in keyData.split('\n'):
if pubKey in pair: if pubKey in pair:
privKey = pair.split(',')[1] privKey = pair.split(',')[1]
return privKey return privKey
def changeActiveKey(self, pubKey):
'''Change crypto.pubKey and crypto.privKey to a given key pair by specifying the public key'''
if not pubKey in self.getPubkeyList():
raise ValueError('That pubkey does not exist')
self.crypto.pubKey = pubKey
self.crypto.privKey = self.getPrivkey(pubKey)

15
onionr/onionr.py
View File

@ -34,16 +34,17 @@ from utils import detectoptimization
if detectoptimization.detect_optimization(): if detectoptimization.detect_optimization():
sys.stderr.write('Error, Onionr cannot be run in optimized mode\n') sys.stderr.write('Error, Onionr cannot be run in optimized mode\n')
sys.exit(1) sys.exit(1)
import os, base64, random, shutil, time, platform, signal import os, base64, random, shutil, time, platform, signal
from threading import Thread from threading import Thread
import core, config, logger, onionrplugins as plugins, onionrevents as events import config, logger, onionrplugins as plugins, onionrevents as events
import netcontroller import netcontroller
from netcontroller import NetController from netcontroller import NetController
from onionrblockapi import Block from onionrblockapi import Block
import onionrproofs, onionrexceptions, communicator, setupconfig import onionrproofs, onionrexceptions, communicator, setupconfig
import onionrcommands as commands # Many command definitions are here import onionrcommands as commands # Many command definitions are here
from utils import identifyhome from utils import identifyhome
from coredb import keydb
import filepaths
try: try:
from urllib3.contrib.socks import SOCKSProxyManager from urllib3.contrib.socks import SOCKSProxyManager
@ -103,8 +104,6 @@ class Onionr:
plugins.disable(name, onionr = self, stop_event = False) plugins.disable(name, onionr = self, stop_event = False)
self.communicatorInst = None self.communicatorInst = None
self.onionrCore = core.Core()
self.onionrCore.onionrInst = self
#self.deleteRunFiles() #self.deleteRunFiles()
self.clientAPIInst = '' # Client http api instance self.clientAPIInst = '' # Client http api instance
@ -168,11 +167,11 @@ class Onionr:
def deleteRunFiles(self): def deleteRunFiles(self):
try: try:
os.remove(self.onionrCore.publicApiHostFile) os.remove(filepaths.public_API_host_file)
except FileNotFoundError: except FileNotFoundError:
pass pass
try: try:
os.remove(self.onionrCore.privateApiHostFile) os.remove(filepaths.private_API_host_file)
except FileNotFoundError: except FileNotFoundError:
pass pass
@ -236,7 +235,7 @@ class Onionr:
def listPeers(self): def listPeers(self):
logger.info('Peer transport address list:') logger.info('Peer transport address list:')
for i in self.onionrCore.listAdders(): for i in keydb.listkeys.list_adders():
logger.info(i, terminal=True) logger.info(i, terminal=True)
def getWebPassword(self): def getWebPassword(self):
@ -304,7 +303,7 @@ class Onionr:
''' '''
Displays a list of keys (used to be called peers) (?) Displays a list of keys (used to be called peers) (?)
''' '''
logger.info('%sPublic keys in database: \n%s%s' % (logger.colors.fg.lightgreen, logger.colors.fg.green, '\n'.join(self.onionrCore.listPeers())), terminal=True) logger.info('%sPublic keys in database: \n%s%s' % (logger.colors.fg.lightgreen, logger.colors.fg.green, '\n'.join(keydb.listkeys.list_peers()())), terminal=True)
def addPeer(self): def addPeer(self):
''' '''

16
onionr/onionrblacklist.py
View File

@ -17,19 +17,21 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import sqlite3, os, logger import sqlite3, os
import logger, onionrcrypto
from onionrutils import epoch, bytesconverter from onionrutils import epoch, bytesconverter
from coredb import dbfiles
crypto = onionrcrypto.OnionrCrypto()
class OnionrBlackList: class OnionrBlackList:
def __init__(self, coreInst): def __init__(self):
self.blacklistDB = coreInst.dataDir + 'blacklist.db' self.blacklistDB = dbfiles.blacklist_db
self._core = coreInst
if not os.path.exists(self.blacklistDB): if not os.path.exists(dbfiles.blacklist_db):
self.generateDB() self.generateDB()
return return
def inBlacklist(self, data): def inBlacklist(self, data):
hashed = bytesconverter.bytes_to_str(self._core._crypto.sha3Hash(data)) hashed = bytesconverter.bytes_to_str(crypto.sha3Hash(data))
retData = False retData = False
if not hashed.isalnum(): if not hashed.isalnum():
@ -99,7 +101,7 @@ class OnionrBlackList:
2=pubkey 2=pubkey
''' '''
# we hash the data so we can remove data entirely from our node's disk # we hash the data so we can remove data entirely from our node's disk
hashed = bytesconverter.bytes_to_str(self._core._crypto.sha3Hash(data)) hashed = bytesconverter.bytes_to_str(crypto.sha3Hash(data))
if len(hashed) > 64: if len(hashed) > 64:
raise Exception("Hashed data is too large") raise Exception("Hashed data is too large")

87
onionr/onionrblockapi.py
View File

@ -18,23 +18,25 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import core as onionrcore, logger, config, onionrexceptions, nacl.exceptions import logger, config, onionrexceptions, nacl.exceptions
import json, os, sys, datetime, base64, onionrstorage import json, os, sys, datetime, base64, onionrstorage, onionrcrypto
from onionrusers import onionrusers from onionrusers import onionrusers
from onionrutils import stringvalidators, epoch from onionrutils import stringvalidators, epoch
from coredb import blockmetadb from coredb import blockmetadb
from onionrstorage import removeblock
import onionrblocks
class Block: class Block:
crypto = onionrcrypto.OnionrCrypto()
blockCacheOrder = list() # NEVER write your own code that writes to this! blockCacheOrder = list() # NEVER write your own code that writes to this!
blockCache = dict() # should never be accessed directly, look at Block.getCache() blockCache = dict() # should never be accessed directly, look at Block.getCache()
def __init__(self, hash = None, core = None, type = None, content = None, expire=None, decrypt=False, bypassReplayCheck=False): def __init__(self, hash = None, type = None, content = None, expire=None, decrypt=False, bypassReplayCheck=False):
# take from arguments # take from arguments
# sometimes people input a bytes object instead of str in `hash` # sometimes people input a bytes object instead of str in `hash`
if (not hash is None) and isinstance(hash, bytes): if (not hash is None) and isinstance(hash, bytes):
hash = hash.decode() hash = hash.decode()
self.hash = hash self.hash = hash
self.core = core
self.btype = type self.btype = type
self.bcontent = content self.bcontent = content
self.expire = expire self.expire = expire
@ -56,10 +58,6 @@ class Block:
self.validSig = False self.validSig = False
self.autoDecrypt = decrypt self.autoDecrypt = decrypt
# handle arguments
if self.getCore() is None:
self.core = onionrcore.Core()
self.update() self.update()
def decrypt(self, encodedData = True): def decrypt(self, encodedData = True):
@ -70,27 +68,26 @@ class Block:
if self.decrypted: if self.decrypted:
return True return True
retData = False retData = False
core = self.getCore()
# decrypt data # decrypt data
if self.getHeader('encryptType') == 'asym': if self.getHeader('encryptType') == 'asym':
try: try:
self.bcontent = core._crypto.pubKeyDecrypt(self.bcontent, encodedData=encodedData) self.bcontent = crypto.pubKeyDecrypt(self.bcontent, encodedData=encodedData)
bmeta = core._crypto.pubKeyDecrypt(self.bmetadata, encodedData=encodedData) bmeta = crypto.pubKeyDecrypt(self.bmetadata, encodedData=encodedData)
try: try:
bmeta = bmeta.decode() bmeta = bmeta.decode()
except AttributeError: except AttributeError:
# yet another bytes fix # yet another bytes fix
pass pass
self.bmetadata = json.loads(bmeta) self.bmetadata = json.loads(bmeta)
self.signature = core._crypto.pubKeyDecrypt(self.signature, encodedData=encodedData) self.signature = crypto.pubKeyDecrypt(self.signature, encodedData=encodedData)
self.signer = core._crypto.pubKeyDecrypt(self.signer, encodedData=encodedData) self.signer = crypto.pubKeyDecrypt(self.signer, encodedData=encodedData)
self.bheader['signer'] = self.signer.decode() self.bheader['signer'] = self.signer.decode()
self.signedData = json.dumps(self.bmetadata) + self.bcontent.decode() self.signedData = json.dumps(self.bmetadata) + self.bcontent.decode()
# Check for replay attacks # Check for replay attacks
try: try:
if epoch.get_epoch() - blockmetadb.get_block_date(self.hash) > 60: if epoch.get_epoch() - blockmetadb.get_block_date(self.hash) > 60:
assert self.core._crypto.replayTimestampValidation(self.bmetadata['rply']) assert self.crypto.replayTimestampValidation(self.bmetadata['rply'])
except (AssertionError, KeyError, TypeError) as e: except (AssertionError, KeyError, TypeError) as e:
if not self.bypassReplayCheck: if not self.bypassReplayCheck:
# Zero out variables to prevent reading of replays # Zero out variables to prevent reading of replays
@ -106,7 +103,7 @@ class Block:
pass pass
else: else:
try: try:
self.bcontent = onionrusers.OnionrUser(self.getCore(), self.signer).forwardDecrypt(self.bcontent) self.bcontent = onionrusers.OnionrUser(self.signer).forwardDecrypt(self.bcontent)
except (onionrexceptions.DecryptionError, nacl.exceptions.CryptoError) as e: except (onionrexceptions.DecryptionError, nacl.exceptions.CryptoError) as e:
logger.error(str(e)) logger.error(str(e))
pass pass
@ -127,9 +124,7 @@ class Block:
Verify if a block's signature is signed by its claimed signer Verify if a block's signature is signed by its claimed signer
''' '''
core = self.getCore() if crypto.edVerify(data=self.signedData, key=self.signer, sig=self.signature, encodedData=True):
if core._crypto.edVerify(data=self.signedData, key=self.signer, sig=self.signature, encodedData=True):
self.validSig = True self.validSig = True
else: else:
self.validSig = False self.validSig = False
@ -158,7 +153,7 @@ class Block:
# import from file # import from file
if blockdata is None: if blockdata is None:
try: try:
blockdata = onionrstorage.getData(self.core, self.getHash()).decode() blockdata = onionrstorage.getData(self.getHash()).decode()
except AttributeError: except AttributeError:
raise onionrexceptions.NoDataAvailable('Block does not exist') raise onionrexceptions.NoDataAvailable('Block does not exist')
else: else:
@ -220,7 +215,8 @@ class Block:
os.remove(self.getBlockFile()) os.remove(self.getBlockFile())
except TypeError: except TypeError:
pass pass
self.getCore().removeBlock(self.getHash())
removeblock.remove_block(self.getHash())
return True return True
return False return False
@ -238,14 +234,8 @@ class Block:
try: try:
if self.isValid() is True: if self.isValid() is True:
'''
if (not self.getBlockFile() is None) and (recreate is True): self.hash = onionrblocks.insert(self.getRaw(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire())
onionrstorage.store(self.core, self.getRaw().encode())
#with open(self.getBlockFile(), 'wb') as blockFile:
# blockFile.write(self.getRaw().encode())
else:
'''
self.hash = self.getCore().insertBlock(self.getRaw(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire())
if self.hash != False: if self.hash != False:
self.update() self.update()
@ -278,16 +268,6 @@ class Block:
return self.hash return self.hash
def getCore(self):
'''
Returns the Core instance being used by the Block
Outputs:
- (Core): the Core instance
'''
return self.core
def getType(self): def getType(self):
''' '''
Returns the type of the block Returns the type of the block
@ -363,7 +343,7 @@ class Block:
if self.parent == self.getHash(): if self.parent == self.getHash():
self.parent = self self.parent = self
elif Block.exists(self.parent): elif Block.exists(self.parent):
self.parent = Block(self.getMetadata('parent'), core = self.getCore()) self.parent = Block(self.getMetadata('parent'))
else: else:
self.parent = None self.parent = None
@ -445,7 +425,7 @@ class Block:
if (not self.isSigned()) or (not stringvalidators.validate_pub_key(signer)): if (not self.isSigned()) or (not stringvalidators.validate_pub_key(signer)):
return False return False
return bool(self.getCore()._crypto.edVerify(self.getSignedData(), signer, self.getSignature(), encodedData = encodedData)) return bool(crypto.edVerify(self.getSignedData(), signer, self.getSignature(), encodedData = encodedData))
except: except:
return False return False
@ -511,7 +491,7 @@ class Block:
''' '''
if type(parent) == str: if type(parent) == str:
parent = Block(parent, core = self.getCore()) parent = Block(parent)
self.parent = parent self.parent = parent
self.setMetadata('parent', (None if parent is None else self.getParent().getHash())) self.setMetadata('parent', (None if parent is None else self.getParent().getHash()))
@ -519,7 +499,7 @@ class Block:
# static functions # static functions
def getBlocks(type = None, signer = None, signed = None, parent = None, reverse = False, limit = None, core = None): def getBlocks(type = None, signer = None, signed = None, parent = None, reverse = False, limit = None):
''' '''
Returns a list of Block objects based on supplied filters Returns a list of Block objects based on supplied filters
@ -528,24 +508,22 @@ class Block:
- signer (str/list): filters by signer (one in the list has to be a signer) - signer (str/list): filters by signer (one in the list has to be a signer)
- signed (bool): filters out by whether or not the block is signed - signed (bool): filters out by whether or not the block is signed
- reverse (bool): reverses the list if True - reverse (bool): reverses the list if True
- core (Core): lets you optionally supply a core instance so one doesn't need to be started
Outputs: Outputs:
- (list): a list of Block objects that match the input - (list): a list of Block objects that match the input
''' '''
try: try:
core = (core if not core is None else onionrcore.Core())
if (not parent is None) and (not isinstance(parent, Block)): if (not parent is None) and (not isinstance(parent, Block)):
parent = Block(hash = parent, core = core) parent = Block(hash = parent)
relevant_blocks = list() relevant_blocks = list()
blocks = (blockmetadb.get_block_list() if type is None else blockmetadb.get_blocks_by_type(type)) blocks = (blockmetadb.get_block_list() if type is None else blockmetadb.get_blocks_by_type(type))
for block in blocks: for block in blocks:
if Block.exists(block): if Block.exists(block):
block = Block(block, core = core) block = Block(block)
relevant = True relevant = True
@ -586,7 +564,7 @@ class Block:
return list() return list()
def mergeChain(child, file = None, maximumFollows = 1000, core = None): def mergeChain(child, file = None, maximumFollows = 1000):
''' '''
Follows a child Block to its root parent Block, merging content Follows a child Block to its root parent Block, merging content
@ -596,8 +574,6 @@ class Block:
- maximumFollows (int): the maximum number of Blocks to follow - maximumFollows (int): the maximum number of Blocks to follow
''' '''
# validate data and instantiate Core
core = (core if not core is None else onionrcore.Core())
maximumFollows = max(0, maximumFollows) maximumFollows = max(0, maximumFollows)
# type conversions # type conversions
@ -617,7 +593,7 @@ class Block:
if len(blocks) - 1 >= maximumFollows: if len(blocks) - 1 >= maximumFollows:
break break
block = Block(blocks[-1], core = core).getParent() block = Block(blocks[-1]).getParent()
# end if there is no parent Block # end if there is no parent Block
if block is None: if block is None:
@ -637,7 +613,7 @@ class Block:
# combine block contents # combine block contents
for hash in blocks: for hash in blocks:
block = Block(hash, core = core) block = Block(hash)
contents = block.getContent() contents = block.getContent()
contents = base64.b64decode(contents.encode()) contents = base64.b64decode(contents.encode())
@ -669,16 +645,11 @@ class Block:
# no input data? scrap it. # no input data? scrap it.
if bHash is None: if bHash is None:
return False return False
'''
if type(hash) == Block:
blockfile = hash.getBlockFile()
else:
blockfile = onionrcore.Core().dataDir + 'blocks/%s.dat' % hash
'''
if isinstance(bHash, Block): if isinstance(bHash, Block):
bHash = bHash.getHash() bHash = bHash.getHash()
ret = isinstance(onionrstorage.getData(onionrcore.Core(), bHash), type(None)) ret = isinstance(onionrstorage.getData(bHash), type(None))
return not ret return not ret

12
onionr/onionrblocks/insert.py
View File

@ -1,17 +1,17 @@
import json import json
from onionrutils import bytesconverter, epoch from onionrutils import bytesconverter, epoch
import storagecounter, filepaths, onionrvalues, onionrstorage import storagecounter, filepaths, onionrstorage
import onionrevents as events import onionrevents as events
from etc import powchoice from etc import powchoice, onionrvalues
crypto = onionrcrypto.OnionrCrypto() def insert_block(onionr_inst, data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False):
use_subprocess = powchoice.use_subprocess()
def insert_block(data, header='txt', sign=False, encryptType='', symKey='', asymPeer='', meta = {}, expire=None, disableForward=False):
''' '''
Inserts a block into the network Inserts a block into the network
encryptType must be specified to encrypt a block encryptType must be specified to encrypt a block
''' '''
use_subprocess = powchoice.use_subprocess(onionr_inst.config)
requirements = onionrvalues.OnionrValues() requirements = onionrvalues.OnionrValues()
storage_counter = storagecounter.StorageCounter() storage_counter = storagecounter.StorageCounter()
crypto = onionr_inst.crypto
allocationReachedMessage = 'Cannot insert block, disk allocation reached.' allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
if storage_counter.isFull(): if storage_counter.isFull():
logger.error(allocationReachedMessage) logger.error(allocationReachedMessage)
@ -23,7 +23,7 @@ def insert_block(data, header='txt', sign=False, encryptType='', symKey='', asym
createTime = epoch.get_epoch() createTime = epoch.get_epoch()
dataNonce = bytesconverter.bytes_to_str(crypto.sha3Hash(data)) dataNonce = bytesconverter.bytes_to_str(hashers.sha3_hash(data))
try: try:
with open(filepaths.data_nonce_file, 'r') as nonces: with open(filepaths.data_nonce_file, 'r') as nonces:
if dataNonce in nonces: if dataNonce in nonces:

19
onionr/onionrcrypto/__init__.py
View File

@ -25,6 +25,7 @@ from onionrutils import stringvalidators, epoch, bytesconverter
import filepaths import filepaths
import onionrexceptions, keymanager, onionrutils import onionrexceptions, keymanager, onionrutils
import config import config
from . import generate, hashers
config.reload() config.reload()
class OnionrCrypto: class OnionrCrypto:
@ -175,9 +176,7 @@ class OnionrCrypto:
def generatePubKey(self): def generatePubKey(self):
'''Generate a Ed25519 public key pair, return tuple of base32encoded pubkey, privkey''' '''Generate a Ed25519 public key pair, return tuple of base32encoded pubkey, privkey'''
private_key = nacl.signing.SigningKey.generate() return generate.generate_pub_key()
public_key = private_key.verify_key.encode(encoder=nacl.encoding.Base32Encoder())
return (public_key.decode(), private_key.encode(encoder=nacl.encoding.Base32Encoder()).decode())
def generateDeterministic(self, passphrase, bypassCheck=False): def generateDeterministic(self, passphrase, bypassCheck=False):
'''Generate a Ed25519 public key pair from a password''' '''Generate a Ed25519 public key pair from a password'''
@ -215,20 +214,10 @@ class OnionrCrypto:
return result return result
def sha3Hash(self, data): def sha3Hash(self, data):
try: return hashers.sha3_hash(data)
data = data.encode()
except AttributeError:
pass
hasher = hashlib.sha3_256()
hasher.update(data)
return hasher.hexdigest()
def blake2bHash(self, data): def blake2bHash(self, data):
try: return hashers.blake2b_hash(data)
data = data.encode()
except AttributeError:
pass
return nacl.hash.blake2b(data)
def verifyPow(self, blockContent): def verifyPow(self, blockContent):
''' '''

6
onionr/onionrcrypto/generate.py Normal file
View File

@ -0,0 +1,6 @@
import nacl.signing, nacl.encoding
def generate_pub_key():
'''Generate a Ed25519 public key pair, return tuple of base32encoded pubkey, privkey'''
private_key = nacl.signing.SigningKey.generate()
public_key = private_key.verify_key.encode(encoder=nacl.encoding.Base32Encoder())
return (public_key.decode(), private_key.encode(encoder=nacl.encoding.Base32Encoder()).decode())

16
onionr/onionrcrypto/hashers.py Normal file
View File

@ -0,0 +1,16 @@
import hashlib, nacl.hash
def sha3_hash(data):
try:
data = data.encode()
except AttributeError:
pass
hasher = hashlib.sha3_256()
hasher.update(data)
return hasher.hexdigest()
def blake2b_hash(data):
try:
data = data.encode()
except AttributeError:
pass
return nacl.hash.blake2b(data)

19
onionr/onionrpeers/peercleanup.py
View File

@ -21,12 +21,13 @@ import sqlite3
import logger import logger
from onionrutils import epoch from onionrutils import epoch
from . import scoresortedpeerlist, peerprofiles from . import scoresortedpeerlist, peerprofiles
def peer_cleanup(core_inst): import onionrblacklist
from coredb import keydb
def peer_cleanup(onionr_inst):
'''Removes peers who have been offline too long or score too low''' '''Removes peers who have been offline too long or score too low'''
config = core_inst.config
logger.info('Cleaning peers...') logger.info('Cleaning peers...')
blacklist = onionrblacklist.OnionrBlackList()
adders = scoresortedpeerlist.get_score_sorted_peer_list(core_inst) adders = scoresortedpeerlist.get_score_sorted_peer_list()
adders.reverse() adders.reverse()
if len(adders) > 1: if len(adders) > 1:
@ -36,14 +37,14 @@ def peer_cleanup(core_inst):
for address in adders: for address in adders:
# Remove peers that go below the negative score # Remove peers that go below the negative score
if peerprofiles.PeerProfiles(address, core_inst).score < min_score: if peerprofiles.PeerProfiles(address).score < min_score:
core_inst.removeAddress(address) keydb.removekeys.remove_address(address)
try: try:
if (int(epoch.get_epoch()) - int(core_inst.getPeerInfo(address, 'dateSeen'))) >= 600: if (int(epoch.get_epoch()) - int(keydb.transportinfo.get_address_info(address, 'dateSeen'))) >= 600:
expireTime = 600 expireTime = 600
else: else:
expireTime = 86400 expireTime = 86400
core_inst._blacklist.addToDB(address, dataType=1, expire=expireTime) blacklist.addToDB(address, dataType=1, expire=expireTime)
except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue except sqlite3.IntegrityError: #TODO just make sure its not a unique constraint issue
pass pass
except ValueError: except ValueError:
@ -51,4 +52,4 @@ def peer_cleanup(core_inst):
logger.warn('Removed address ' + address + '.') logger.warn('Removed address ' + address + '.')
# Unban probably not malicious peers TODO improve # Unban probably not malicious peers TODO improve
core_inst._blacklist.deleteExpired(dataType=1) blacklist.deleteExpired(dataType=1)

4
onionr/onionrpeers/peerprofiles.py
View File

@ -21,7 +21,7 @@ class PeerProfiles:
''' '''
PeerProfiles PeerProfiles
''' '''
def __init__(self, address, coreInst): def __init__(self, address):
self.address = address # node address self.address = address # node address
self.score = None self.score = None
self.friendSigCount = 0 self.friendSigCount = 0
@ -29,8 +29,6 @@ class PeerProfiles:
self.failure = 0 self.failure = 0
self.connectTime = None self.connectTime = None
self.coreInst = coreInst
self.loadScore() self.loadScore()
self.getConnectTime() self.getConnectTime()
return return

7
onionr/onionrpeers/scoresortedpeerlist.py
View File

@ -18,14 +18,15 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
from . import peerprofiles from . import peerprofiles
def get_score_sorted_peer_list(coreInst): from coredb import keydb
peer_list = coreInst.listAdders() def get_score_sorted_peer_list():
peer_list = keydb.listkeys.list_adders()
peer_scores = {} peer_scores = {}
peer_times = {} peer_times = {}
for address in peer_list: for address in peer_list:
# Load peer's profiles into a list # Load peer's profiles into a list
profile = peerprofiles.PeerProfiles(address, coreInst) profile = peerprofiles.PeerProfiles(address)
peer_scores[address] = profile.score peer_scores[address] = profile.score
if not isinstance(profile.connectTime, type(None)): if not isinstance(profile.connectTime, type(None)):
peer_times[address] = profile.connectTime peer_times[address] = profile.connectTime

14
onionr/onionrpluginapi.py
View File

@ -18,7 +18,7 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import onionrplugins, core as onionrcore, logger import onionrplugins, logger, onionrcrypto
from onionrutils import localcommand from onionrutils import localcommand
from coredb import daemonqueue from coredb import daemonqueue
class DaemonAPI: class DaemonAPI:
@ -39,7 +39,7 @@ class DaemonAPI:
return daemonqueue.daemon_queue_add(command, data) return daemonqueue.daemon_queue_add(command, data)
def local_command(self, command): def local_command(self, command):
return localcommand.local_command(self.pluginapi.get_core(), command) return localcommand.local_command(command)
def queue_pop(self): def queue_pop(self):
return daemonqueue.daemon_queue() return daemonqueue.daemon_queue()
@ -149,15 +149,12 @@ class pluginapi:
def __init__(self, onionr, data): def __init__(self, onionr, data):
self.onionr = onionr self.onionr = onionr
self.data = data self.data = data
if self.onionr is None:
self.core = onionrcore.Core()
else:
self.core = self.onionr.onionrCore
self.daemon = DaemonAPI(self) self.daemon = DaemonAPI(self)
self.plugins = PluginAPI(self) self.plugins = PluginAPI(self)
self.commands = CommandAPI(self) self.commands = CommandAPI(self)
self.web = WebAPI(self) self.web = WebAPI(self)
self.crypto = onionrcrypto.OnionrCrypto()
def get_onionr(self): def get_onionr(self):
return self.onionr return self.onionr
@ -165,11 +162,8 @@ class pluginapi:
def get_data(self): def get_data(self):
return self.data return self.data
def get_core(self):
return self.core
def get_crypto(self): def get_crypto(self):
return self.get_core()._crypto return self.crypto
def get_daemonapi(self): def get_daemonapi(self):
return self.daemon return self.daemon

42
onionr/onionrproofs.py
View File

@ -18,19 +18,18 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import multiprocessing, nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, sys, json import multiprocessing, nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, sys, json
import core, config, logger, onionrblockapi import config, logger, onionrblockapi, storagecounter, onionrcrypto
from onionrutils import bytesconverter from onionrutils import bytesconverter
config.reload() config.reload()
crypto = onionrcrypto.OnionrCrypto()
def getDifficultyModifier(coreOrUtilsInst=None): def getDifficultyModifier():
'''Accepts a core or utils instance returns '''returns the difficulty modifier for block storage based
the difficulty modifier for block storage based
on a variety of factors, currently only disk use. on a variety of factors, currently only disk use.
''' '''
classInst = coreOrUtilsInst classInst = coreOrUtilsInst
retData = 0 retData = 0
useFunc = classInst.storage_counter.getPercent useFunc = storagecounter.StorageCounter().getPercent
percentUse = useFunc() percentUse = useFunc()
@ -43,7 +42,7 @@ def getDifficultyModifier(coreOrUtilsInst=None):
return retData return retData
def getDifficultyForNewBlock(data, ourBlock=True, coreInst=None): def getDifficultyForNewBlock(data, ourBlock=True):
''' '''
Get difficulty for block. Accepts size in integer, Block instance, or str/bytes full block contents Get difficulty for block. Accepts size in integer, Block instance, or str/bytes full block contents
''' '''
@ -59,7 +58,7 @@ def getDifficultyForNewBlock(data, ourBlock=True, coreInst=None):
else: else:
minDifficulty = config.get('general.minimum_block_pow', 4) minDifficulty = config.get('general.minimum_block_pow', 4)
retData = max(minDifficulty, math.floor(dataSize / 100000)) + getDifficultyModifier(coreInst) retData = max(minDifficulty, math.floor(dataSize / 100000)) + getDifficultyModifier()
return retData return retData
@ -118,12 +117,11 @@ class DataPOW:
self.mainHash = '0' * 70 self.mainHash = '0' * 70
self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))] self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))]
myCore = core.Core()
for i in range(max(1, threadCount)): for i in range(max(1, threadCount)):
t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore)) t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True))
t.start() t.start()
def pow(self, reporting = False, myCore = None): def pow(self, reporting = False):
startTime = math.floor(time.time()) startTime = math.floor(time.time())
self.hashing = True self.hashing = True
self.reporting = reporting self.reporting = reporting
@ -187,20 +185,13 @@ class DataPOW:
return result return result
class POW: class POW:
def __init__(self, metadata, data, threadCount = 1, forceDifficulty=0, coreInst=None): def __init__(self, metadata, data, threadCount = 1, forceDifficulty=0):
self.foundHash = False self.foundHash = False
self.difficulty = 0 self.difficulty = 0
self.data = data self.data = data
self.metadata = metadata self.metadata = metadata
self.threadCount = threadCount self.threadCount = threadCount
try:
assert isinstance(coreInst, core.Core)
except AssertionError:
myCore = core.Core()
else:
myCore = coreInst
json_metadata = json.dumps(metadata).encode() json_metadata = json.dumps(metadata).encode()
try: try:
@ -212,21 +203,18 @@ class POW:
self.difficulty = forceDifficulty self.difficulty = forceDifficulty
else: else:
# Calculate difficulty. Dumb for now, may use good algorithm in the future. # Calculate difficulty. Dumb for now, may use good algorithm in the future.
self.difficulty = getDifficultyForNewBlock(bytes(json_metadata + b'\n' + self.data), coreInst=myCore) self.difficulty = getDifficultyForNewBlock(bytes(json_metadata + b'\n' + self.data))
logger.info('Computing POW (difficulty: %s)...' % (self.difficulty,))
logger.info('Computing POW (difficulty: %s)...' % self.difficulty)
self.mainHash = '0' * 64 self.mainHash = '0' * 64
self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))] self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))]
for i in range(max(1, threadCount)): for i in range(max(1, threadCount)):
t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore)) t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True))
t.start() t.start()
self.myCore = myCore
return
def pow(self, reporting = False, myCore = None): def pow(self, reporting = False):
startTime = math.floor(time.time()) startTime = math.floor(time.time())
self.hashing = True self.hashing = True
self.reporting = reporting self.reporting = reporting
@ -239,7 +227,7 @@ class POW:
#token = nacl.hash.blake2b(rand + self.data).decode() #token = nacl.hash.blake2b(rand + self.data).decode()
self.metadata['pow'] = nonce self.metadata['pow'] = nonce
payload = json.dumps(self.metadata).encode() + b'\n' + self.data payload = json.dumps(self.metadata).encode() + b'\n' + self.data
token = myCore._crypto.sha3Hash(payload) token = crypto.sha3Hash(payload)
try: try:
# on some versions, token is bytes # on some versions, token is bytes
token = token.decode() token = token.decode()

13
onionr/onionrservices/__init__.py
View File

@ -19,17 +19,14 @@
''' '''
import time import time
import stem import stem
import core
from . import connectionserver, bootstrapservice from . import connectionserver, bootstrapservice
from onionrutils import stringvalidators, basicrequests from onionrutils import stringvalidators, basicrequests
import config
class OnionrServices: class OnionrServices:
''' '''
Create a client or server for connecting to peer interfaces Create a client or server for connecting to peer interfaces
''' '''
def __init__(self, onionr_core): def __init__(self):
assert isinstance(onionr_core, core.Core)
self._core = onionr_core
self.servers = {} self.servers = {}
self.clients = {} self.clients = {}
self.shutdown = False self.shutdown = False
@ -45,11 +42,11 @@ class OnionrServices:
TRY_WAIT = 3 # Seconds to wait before trying bootstrap again TRY_WAIT = 3 # Seconds to wait before trying bootstrap again
# HTTP is fine because .onion/i2p is encrypted/authenticated # HTTP is fine because .onion/i2p is encrypted/authenticated
base_url = 'http://%s/' % (address,) base_url = 'http://%s/' % (address,)
socks = self._core.config.get('tor.socksport') socks = config.get('tor.socksport')
for x in range(BOOTSTRAP_TRIES): for x in range(BOOTSTRAP_TRIES):
if basicrequests.do_get_request(self._core, base_url + 'ping', port=socks, ignoreAPI=True) == 'pong!': if basicrequests.do_get_request(base_url + 'ping', port=socks, ignoreAPI=True) == 'pong!':
# if bootstrap sever is online, tell them our service address # if bootstrap sever is online, tell them our service address
connectionserver.ConnectionServer(peer, address, core_inst=self._core) connectionserver.ConnectionServer(peer, address)
else: else:
time.sleep(TRY_WAIT) time.sleep(TRY_WAIT)
else: else:

15
onionr/onionrservices/bootstrapservice.py
View File

@ -21,17 +21,14 @@ import time, threading, uuid
from gevent.pywsgi import WSGIServer, WSGIHandler from gevent.pywsgi import WSGIServer, WSGIHandler
from stem.control import Controller from stem.control import Controller
from flask import Flask, Response from flask import Flask, Response
import core
from netcontroller import get_open_port from netcontroller import get_open_port
from . import httpheaders from . import httpheaders
from onionrutils import stringvalidators, epoch from onionrutils import stringvalidators, epoch
def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300): def bootstrap_client_service(peer, onionr_inst=None, bootstrap_timeout=300):
''' '''
Bootstrap client services Bootstrap client services
''' '''
if core_inst is None:
core_inst = core.Core()
if not stringvalidators.validate_pub_key(peer): if not stringvalidators.validate_pub_key(peer):
raise ValueError('Peer must be valid base32 ed25519 public key') raise ValueError('Peer must be valid base32 ed25519 public key')
@ -40,11 +37,11 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300):
bootstrap_app = Flask(__name__) bootstrap_app = Flask(__name__)
http_server = WSGIServer(('127.0.0.1', bootstrap_port), bootstrap_app, log=None) http_server = WSGIServer(('127.0.0.1', bootstrap_port), bootstrap_app, log=None)
try: try:
assert core_inst.onionrInst.communicatorInst is not None assert onionr_inst.communicatorInst is not None
except (AttributeError, AssertionError) as e: except (AttributeError, AssertionError) as e:
pass pass
else: else:
core_inst.onionrInst.communicatorInst.service_greenlets.append(http_server) onionr_inst.communicatorInst.service_greenlets.append(http_server)
bootstrap_address = '' bootstrap_address = ''
shutdown = False shutdown = False
@ -71,9 +68,9 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300):
else: else:
return Response("") return Response("")
with Controller.from_port(port=core_inst.config.get('tor.controlPort')) as controller: with Controller.from_port(port=onionr_inst.config.get('tor.controlPort')) as controller:
# Connect to the Tor process for Onionr # Connect to the Tor process for Onionr
controller.authenticate(core_inst.config.get('tor.controlpassword')) controller.authenticate(onionr_inst.config.get('tor.controlpassword'))
# Create the v3 onion service # Create the v3 onion service
response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, key_type = 'NEW', key_content = 'ED25519-V3', await_publication = True) response = controller.create_ephemeral_hidden_service({80: bootstrap_port}, key_type = 'NEW', key_content = 'ED25519-V3', await_publication = True)
core_inst.insertBlock(response.service_id, header='con', sign=True, encryptType='asym', core_inst.insertBlock(response.service_id, header='con', sign=True, encryptType='asym',
@ -86,4 +83,4 @@ def bootstrap_client_service(peer, core_inst=None, bootstrap_timeout=300):
# This line reached when server is shutdown by being bootstrapped # This line reached when server is shutdown by being bootstrapped
# Now that the bootstrap server has received a server, return the address # Now that the bootstrap server has received a server, return the address
return core_inst.keyStore.get(bs_id) return onionr_inst.keyStore.get(bs_id)

26
onionr/onionrservices/connectionserver.py
View File

@ -20,7 +20,7 @@
from gevent.pywsgi import WSGIServer from gevent.pywsgi import WSGIServer
from stem.control import Controller from stem.control import Controller
from flask import Flask from flask import Flask
import core, logger, httpapi import logger, httpapi
import onionrexceptions import onionrexceptions
from netcontroller import get_open_port from netcontroller import get_open_port
from httpapi import apiutils from httpapi import apiutils
@ -28,21 +28,17 @@ from onionrutils import stringvalidators, basicrequests, bytesconverter
from . import httpheaders from . import httpheaders
class ConnectionServer: class ConnectionServer:
def __init__(self, peer, address, core_inst=None): def __init__(self, peer, address, onionr_inst=None):
if core_inst is None:
self.core_inst = core.Core()
else:
self.core_inst = core_inst
if not stringvalidators.validate_pub_key(peer): if not stringvalidators.validate_pub_key(peer):
raise ValueError('Peer must be valid base32 ed25519 public key') raise ValueError('Peer must be valid base32 ed25519 public key')
socks = core_inst.config.get('tor.socksport') # Load config for Tor socks port for proxy socks = onionr_inst.config.get('tor.socksport') # Load config for Tor socks port for proxy
service_app = Flask(__name__) # Setup Flask app for server. service_app = Flask(__name__) # Setup Flask app for server.
service_port = get_open_port() service_port = get_open_port()
service_ip = apiutils.setbindip.set_bind_IP(core_inst=self.core_inst) service_ip = apiutils.setbindip.set_bind_IP()
http_server = WSGIServer(('127.0.0.1', service_port), service_app, log=None) http_server = WSGIServer(('127.0.0.1', service_port), service_app, log=None)
core_inst.onionrInst.communicatorInst.service_greenlets.append(http_server) onionr_inst.communicatorInst.service_greenlets.append(http_server)
# TODO define basic endpoints useful for direct connections like stats # TODO define basic endpoints useful for direct connections like stats
@ -54,7 +50,7 @@ class ConnectionServer:
@service_app.route('/close') @service_app.route('/close')
def shutdown_server(): def shutdown_server():
core_inst.onionrInst.communicatorInst.service_greenlets.remove(http_server) onionr_inst.communicatorInst.service_greenlets.remove(http_server)
http_server.stop() http_server.stop()
return Response('goodbye') return Response('goodbye')
@ -64,15 +60,15 @@ class ConnectionServer:
resp = httpheaders.set_default_onionr_http_headers(resp) resp = httpheaders.set_default_onionr_http_headers(resp)
return resp return resp
with Controller.from_port(port=core_inst.config.get('tor.controlPort')) as controller: with Controller.from_port(port=onionr_inst.config.get('tor.controlPort')) as controller:
# Connect to the Tor process for Onionr # Connect to the Tor process for Onionr
controller.authenticate(core_inst.config.get('tor.controlpassword')) controller.authenticate(onionr_inst.config.get('tor.controlpassword'))
# Create the v3 onion service for the peer to connect to # Create the v3 onion service for the peer to connect to
response = controller.create_ephemeral_hidden_service({80: service_port}, await_publication = True, key_type='NEW', key_content = 'ED25519-V3') response = controller.create_ephemeral_hidden_service({80: service_port}, await_publication = True, key_type='NEW', key_content = 'ED25519-V3')
try: try:
for x in range(3): for x in range(3):
attempt = basicrequests.do_post_request(self.core_inst, 'http://' + address + '/bs/' + response.service_id, port=socks) attempt = basicrequests.do_post_request('http://' + address + '/bs/' + response.service_id, port=socks)
if attempt == 'success': if attempt == 'success':
break break
else: else:
@ -82,8 +78,8 @@ class ConnectionServer:
raise ConnectionError('Could not reach %s bootstrap address %s' % (peer, address)) raise ConnectionError('Could not reach %s bootstrap address %s' % (peer, address))
else: else:
# If no connection error, create the service and save it to local global key store # If no connection error, create the service and save it to local global key store
self.core_inst.keyStore.put('dc-' + response.service_id, bytesconverter.bytes_to_str(peer)) self.onionr_inst.keyStore.put('dc-' + response.service_id, bytesconverter.bytes_to_str(peer))
logger.info('hosting on %s with %s' % (response.service_id, peer)) logger.info('hosting on %s with %s' % (response.service_id, peer))
http_server.serve_forever() http_server.serve_forever()
http_server.stop() http_server.stop()
self.core_inst.keyStore.delete('dc-' + response.service_id) self.onionr_inst.keyStore.delete('dc-' + response.service_id)

49
onionr/onionrstorage/__init__.py
View File

@ -17,31 +17,31 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import core, sys, sqlite3, os, dbcreator, onionrexceptions import sys, sqlite3, os
from onionrutils import bytesconverter, stringvalidators from onionrutils import bytesconverter, stringvalidators
from coredb import dbfiles
import filepaths, onionrcrypto, dbcreator, onionrexceptions
from onionrcrypto import hashers
DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
def dbCreate(coreInst): def dbCreate():
try: try:
dbcreator.DBCreator(coreInst).createBlockDataDB() dbcreator.DBCreator().createBlockDataDB()
except FileExistsError: except FileExistsError:
pass pass
def _dbInsert(coreInst, blockHash, data): def _dbInsert(blockHash, data):
assert isinstance(coreInst, core.Core) dbCreate()
dbCreate(coreInst) conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
conn = sqlite3.connect(coreInst.blockDataDB, timeout=10)
c = conn.cursor() c = conn.cursor()
data = (blockHash, data) data = (blockHash, data)
c.execute('INSERT INTO blockData (hash, data) VALUES(?, ?);', data) c.execute('INSERT INTO blockData (hash, data) VALUES(?, ?);', data)
conn.commit() conn.commit()
conn.close() conn.close()
def _dbFetch(coreInst, blockHash): def _dbFetch(blockHash):
assert isinstance(coreInst, core.Core) dbCreate()
dbCreate(coreInst) conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
conn = sqlite3.connect(coreInst.blockDataDB, timeout=10)
c = conn.cursor() c = conn.cursor()
for i in c.execute('SELECT data from blockData where hash = ?', (blockHash,)): for i in c.execute('SELECT data from blockData where hash = ?', (blockHash,)):
return i[0] return i[0]
@ -49,14 +49,13 @@ def _dbFetch(coreInst, blockHash):
conn.close() conn.close()
return None return None
def deleteBlock(coreInst, blockHash): def deleteBlock(blockHash):
# You should call core.removeBlock if you automatically want to remove storage byte count # You should call core.removeBlock if you automatically want to remove storage byte count
assert isinstance(coreInst, core.Core) if os.path.exists('%s/%s.dat' % (filepaths.block_data_location, blockHash)):
if os.path.exists('%s/%s.dat' % (coreInst.blockDataLocation, blockHash)): os.remove('%s/%s.dat' % (filepaths.block_data_location, blockHash))
os.remove('%s/%s.dat' % (coreInst.blockDataLocation, blockHash))
return True return True
dbCreate(coreInst) dbCreate()
conn = sqlite3.connect(coreInst.blockDataDB, timeout=10) conn = sqlite3.connect(dbfiles.block_data_db, timeout=10)
c = conn.cursor() c = conn.cursor()
data = (blockHash,) data = (blockHash,)
c.execute('DELETE FROM blockData where hash = ?', data) c.execute('DELETE FROM blockData where hash = ?', data)
@ -64,23 +63,21 @@ def deleteBlock(coreInst, blockHash):
conn.close() conn.close()
return True return True
def store(coreInst, data, blockHash=''): def store(data, blockHash=''):
assert isinstance(coreInst, core.Core)
assert stringvalidators.validate_hash(blockHash) assert stringvalidators.validate_hash(blockHash)
ourHash = coreInst._crypto.sha3Hash(data) ourHash = hashers.sha3_hash(data)
if blockHash != '': if blockHash != '':
assert ourHash == blockHash assert ourHash == blockHash
else: else:
blockHash = ourHash blockHash = ourHash
if DB_ENTRY_SIZE_LIMIT >= sys.getsizeof(data): if DB_ENTRY_SIZE_LIMIT >= sys.getsizeof(data):
_dbInsert(coreInst, blockHash, data) _dbInsert(blockHash, data)
else: else:
with open('%s/%s.dat' % (coreInst.blockDataLocation, blockHash), 'wb') as blockFile: with open('%s/%s.dat' % (filepaths.block_data_location, blockHash), 'wb') as blockFile:
blockFile.write(data) blockFile.write(data)
def getData(coreInst, bHash): def getData(bHash):
assert isinstance(coreInst, core.Core)
assert stringvalidators.validate_hash(bHash) assert stringvalidators.validate_hash(bHash)
bHash = bytesconverter.bytes_to_str(bHash) bHash = bytesconverter.bytes_to_str(bHash)
@ -89,7 +86,7 @@ def getData(coreInst, bHash):
# if no entry, check disk # if no entry, check disk
# If no entry in either, raise an exception # If no entry in either, raise an exception
retData = None retData = None
fileLocation = '%s/%s.dat' % (coreInst.blockDataLocation, bHash) fileLocation = '%s/%s.dat' % (filepaths.block_data_location, bHash)
if os.path.exists(fileLocation): if os.path.exists(fileLocation):
with open(fileLocation, 'rb') as block: with open(fileLocation, 'rb') as block:
retData = block.read() retData = block.read()

10
onionr/onionrstorage/removeblock.py
View File

@ -1,7 +1,9 @@
import sys, sqlite3 import sys, sqlite3
import onionrexceptions, onionrstorage import onionrexceptions, onionrstorage
from onionrutils import stringvalidators from onionrutils import stringvalidators
def remove_block(core_inst, block): from coredb import dbfiles
import storagecounter
def remove_block(block):
''' '''
remove a block from this node (does not automatically blacklist) remove a block from this node (does not automatically blacklist)
@ -9,13 +11,13 @@ def remove_block(core_inst, block):
''' '''
if stringvalidators.validate_hash(block): if stringvalidators.validate_hash(block):
conn = sqlite3.connect(core_inst.blockDB, timeout=30) conn = sqlite3.connect(dbfiles.block_data_db, timeout=30)
c = conn.cursor() c = conn.cursor()
t = (block,) t = (block,)
c.execute('Delete from hashes where hash=?;', t) c.execute('Delete from hashes where hash=?;', t)
conn.commit() conn.commit()
conn.close() conn.close()
dataSize = sys.getsizeof(onionrstorage.getData(core_inst, block)) dataSize = sys.getsizeof(onionrstorage.getData(block))
core_inst.storage_counter.removeBytes(dataSize) storagecounter.StorageCounter().removeBytes(dataSize)
else: else:
raise onionrexceptions.InvalidHexHash raise onionrexceptions.InvalidHexHash

23
onionr/onionrstorage/setdata.py
View File

@ -1,32 +1,35 @@
import sys, sqlite3 import sys, sqlite3
import onionrstorage, onionrexceptions import onionrstorage, onionrexceptions, onionrcrypto
def set_data(core_inst, data): import filepaths, storagecounter
from coredb import dbfiles
def set_data(data):
''' '''
Set the data assciated with a hash Set the data assciated with a hash
''' '''
crypto = onionrcrypto.OnionrCrypto()
storage_counter = storagecounter.StorageCounter()
data = data data = data
dataSize = sys.getsizeof(data) dataSize = sys.getsizeof(data)
if not type(data) is bytes: if not type(data) is bytes:
data = data.encode() data = data.encode()
dataHash = core_inst._crypto.sha3Hash(data) dataHash = crypto.sha3Hash(data)
if type(dataHash) is bytes: if type(dataHash) is bytes:
dataHash = dataHash.decode() dataHash = dataHash.decode()
blockFileName = core_inst.blockDataLocation + dataHash + '.dat' blockFileName = filepaths.block_data_location + dataHash + '.dat'
try: try:
onionrstorage.getData(core_inst, dataHash) onionrstorage.getData(dataHash)
except onionrexceptions.NoDataAvailable: except onionrexceptions.NoDataAvailable:
if core_inst.storage_counter.addBytes(dataSize) != False: if storage_counter.addBytes(dataSize) != False:
onionrstorage.store(core_inst, data, blockHash=dataHash) onionrstorage.store(data, blockHash=dataHash)
conn = sqlite3.connect(core_inst.blockDB, timeout=30) conn = sqlite3.connect(dbfiles.block_meta_db, timeout=30)
c = conn.cursor() c = conn.cursor()
c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = ?;", (dataHash,)) c.execute("UPDATE hashes SET dataSaved=1 WHERE hash = ?;", (dataHash,))
conn.commit() conn.commit()
conn.close() conn.close()
with open(core_inst.dataNonceFile, 'a') as nonceFile: with open(filepaths.data_nonce_file, 'a') as nonceFile:
nonceFile.write(dataHash + '\n') nonceFile.write(dataHash + '\n')
else: else:
raise onionrexceptions.DiskAllocationReached raise onionrexceptions.DiskAllocationReached

11
onionr/onionrusers/contactmanager.py
View File

@ -21,13 +21,14 @@ import os, json, onionrexceptions
import unpaddedbase32 import unpaddedbase32
from onionrusers import onionrusers from onionrusers import onionrusers
from onionrutils import bytesconverter, epoch from onionrutils import bytesconverter, epoch
from utils import identifyhome
class ContactManager(onionrusers.OnionrUser): class ContactManager(onionrusers.OnionrUser):
def __init__(self, coreInst, publicKey, saveUser=False, recordExpireSeconds=5): def __init__(self, publicKey, saveUser=False, recordExpireSeconds=5):
publicKey = unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)).decode() publicKey = unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)).decode()
super(ContactManager, self).__init__(coreInst, publicKey, saveUser=saveUser) super(ContactManager, self).__init__(publicKey, saveUser=saveUser)
self.dataDir = coreInst.dataDir + '/contacts/' home = identifyhome.identify_home()
self.dataFile = '%s/contacts/%s.json' % (coreInst.dataDir, publicKey) self.dataDir = home + '/contacts/'
self.dataFile = '%s/contacts/%s.json' % (home, publicKey)
self.lastRead = 0 self.lastRead = 0
self.recordExpire = recordExpireSeconds self.recordExpire = recordExpireSeconds
self.data = self._loadData() self.data = self._loadData()

57
onionr/onionrusers/onionrusers.py
View File

@ -21,11 +21,11 @@ import logger, onionrexceptions, json, sqlite3, time
from onionrutils import stringvalidators, bytesconverter, epoch from onionrutils import stringvalidators, bytesconverter, epoch
import unpaddedbase32 import unpaddedbase32
import nacl.exceptions import nacl.exceptions
from coredb import keydb from coredb import keydb, dbfiles
def deleteExpiredKeys(coreInst): def deleteExpiredKeys():
# Fetch the keys we generated for the peer, that are still around # Fetch the keys we generated for the peer, that are still around
conn = sqlite3.connect(coreInst.forwardKeysFile, timeout=10) conn = sqlite3.connect(dbfiles.forward_keys_db, timeout=10)
c = conn.cursor() c = conn.cursor()
curTime = epoch.get_epoch() curTime = epoch.get_epoch()
@ -35,8 +35,8 @@ def deleteExpiredKeys(coreInst):
conn.close() conn.close()
return return
def deleteTheirExpiredKeys(coreInst, pubkey): def deleteTheirExpiredKeys(pubkey):
conn = sqlite3.connect(coreInst.peerDB, timeout=10) conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
c = conn.cursor() c = conn.cursor()
# Prepare the insert # Prepare the insert
@ -51,40 +51,41 @@ DEFAULT_KEY_EXPIRE = 604800
#DEFAULT_KEY_EXPIRE = 600 #DEFAULT_KEY_EXPIRE = 600
class OnionrUser: class OnionrUser:
def __init__(self, coreInst, publicKey, saveUser=False):
def __init__(self, crypto_inst, publicKey, saveUser=False):
''' '''
OnionrUser is an abstraction for "users" of the network. OnionrUser is an abstraction for "users" of the network.
Takes an instance of onionr core, a base32 encoded ed25519 public key, and a bool saveUser Takes a base32 encoded ed25519 public key, and a bool saveUser
saveUser determines if we should add a user to our peer database or not. saveUser determines if we should add a user to our peer database or not.
''' '''
self.crypto = crypto_inst
publicKey = unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)).decode() publicKey = unpaddedbase32.repad(bytesconverter.str_to_bytes(publicKey)).decode()
self.trust = 0 self.trust = 0
self._core = coreInst
self.publicKey = publicKey self.publicKey = publicKey
if saveUser: if saveUser:
try: try:
self._core.addPeer(publicKey) keydb.addkeys.add_peer(publicKey)
except AssertionError: except AssertionError:
pass pass
self.trust = self._core.getPeerInfo(self.publicKey, 'trust') self.trust = keydb.userinfo.get_user_info(self.publicKey, 'trust')
return return
def setTrust(self, newTrust): def setTrust(self, newTrust):
'''Set the peers trust. 0 = not trusted, 1 = friend, 2 = ultimate''' '''Set the peers trust. 0 = not trusted, 1 = friend, 2 = ultimate'''
self._core.setPeerInfo(self.publicKey, 'trust', newTrust) keydb.userinfo.set_user_info(self.publicKey, 'trust', newTrust)
def isFriend(self): def isFriend(self):
if self._core.getPeerInfo(self.publicKey, 'trust') == 1: if keydb.userinfo.set_peer_info(self.publicKey, 'trust') == 1:
return True return True
return False return False
def getName(self): def getName(self):
retData = 'anonymous' retData = 'anonymous'
name = self._core.getPeerInfo(self.publicKey, 'name') name = keydb.userinfo.get_user_info(self.publicKey, 'name')
try: try:
if len(name) > 0: if len(name) > 0:
retData = name retData = name
@ -93,20 +94,20 @@ class OnionrUser:
return retData return retData
def encrypt(self, data): def encrypt(self, data):
encrypted = self._core._crypto.pubKeyEncrypt(data, self.publicKey, encodedData=True) encrypted = self.crypto.pubKeyEncrypt(data, self.publicKey, encodedData=True)
return encrypted return encrypted
def decrypt(self, data): def decrypt(self, data):
decrypted = self._core._crypto.pubKeyDecrypt(data, self.publicKey, encodedData=True) decrypted = self.crypto.pubKeyDecrypt(data, self.publicKey, encodedData=True)
return decrypted return decrypted
def forwardEncrypt(self, data): def forwardEncrypt(self, data):
deleteTheirExpiredKeys(self._core, self.publicKey) deleteTheirExpiredKeys(self.publicKey)
deleteExpiredKeys(self._core) deleteExpiredKeys()
retData = '' retData = ''
forwardKey = self._getLatestForwardKey() forwardKey = self._getLatestForwardKey()
if stringvalidators.validate_pub_key(forwardKey[0]): if stringvalidators.validate_pub_key(forwardKey[0]):
retData = self._core._crypto.pubKeyEncrypt(data, forwardKey[0], encodedData=True) retData = self.crypto.pubKeyEncrypt(data, forwardKey[0], encodedData=True)
else: else:
raise onionrexceptions.InvalidPubkey("No valid forward secrecy key available for this user") raise onionrexceptions.InvalidPubkey("No valid forward secrecy key available for this user")
#self.generateForwardKey() #self.generateForwardKey()
@ -116,7 +117,7 @@ class OnionrUser:
retData = "" retData = ""
for key in self.getGeneratedForwardKeys(False): for key in self.getGeneratedForwardKeys(False):
try: try:
retData = self._core._crypto.pubKeyDecrypt(encrypted, privkey=key[1], encodedData=True) retData = self.crypto.pubKeyDecrypt(encrypted, privkey=key[1], encodedData=True)
except nacl.exceptions.CryptoError: except nacl.exceptions.CryptoError:
retData = False retData = False
else: else:
@ -128,7 +129,7 @@ class OnionrUser:
def _getLatestForwardKey(self): def _getLatestForwardKey(self):
# Get the latest forward secrecy key for a peer # Get the latest forward secrecy key for a peer
key = "" key = ""
conn = sqlite3.connect(self._core.peerDB, timeout=10) conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
c = conn.cursor() c = conn.cursor()
# TODO: account for keys created at the same time (same epoch) # TODO: account for keys created at the same time (same epoch)
@ -142,7 +143,7 @@ class OnionrUser:
return key return key
def _getForwardKeys(self): def _getForwardKeys(self):
conn = sqlite3.connect(self._core.peerDB, timeout=10) conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
c = conn.cursor() c = conn.cursor()
keyList = [] keyList = []
@ -157,11 +158,11 @@ class OnionrUser:
def generateForwardKey(self, expire=DEFAULT_KEY_EXPIRE): def generateForwardKey(self, expire=DEFAULT_KEY_EXPIRE):
# Generate a forward secrecy key for the peer # Generate a forward secrecy key for the peer
conn = sqlite3.connect(self._core.forwardKeysFile, timeout=10) conn = sqlite3.connect(dbfiles.forward_keys_db, timeout=10)
c = conn.cursor() c = conn.cursor()
# Prepare the insert # Prepare the insert
time = epoch.get_epoch() time = epoch.get_epoch()
newKeys = self._core._crypto.generatePubKey() newKeys = self.crypto.generatePubKey()
newPub = bytesconverter.bytes_to_str(newKeys[0]) newPub = bytesconverter.bytes_to_str(newKeys[0])
newPriv = bytesconverter.bytes_to_str(newKeys[1]) newPriv = bytesconverter.bytes_to_str(newKeys[1])
@ -175,7 +176,7 @@ class OnionrUser:
def getGeneratedForwardKeys(self, genNew=True): def getGeneratedForwardKeys(self, genNew=True):
# Fetch the keys we generated for the peer, that are still around # Fetch the keys we generated for the peer, that are still around
conn = sqlite3.connect(self._core.forwardKeysFile, timeout=10) conn = sqlite3.connect(dbfiles.forward_keys_db, timeout=10)
c = conn.cursor() c = conn.cursor()
pubkey = self.publicKey pubkey = self.publicKey
pubkey = bytesconverter.bytes_to_str(pubkey) pubkey = bytesconverter.bytes_to_str(pubkey)
@ -197,7 +198,7 @@ class OnionrUser:
# Do not add if something went wrong with the key # Do not add if something went wrong with the key
raise onionrexceptions.InvalidPubkey(newKey) raise onionrexceptions.InvalidPubkey(newKey)
conn = sqlite3.connect(self._core.peerDB, timeout=10) conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=10)
c = conn.cursor() c = conn.cursor()
# Get the time we're inserting the key at # Get the time we're inserting the key at
@ -222,8 +223,8 @@ class OnionrUser:
return True return True
@classmethod @classmethod
def list_friends(cls, coreInst): def list_friends(cls):
friendList = [] friendList = []
for x in coreInst.listPeers(trust=1): for x in keydb.listkeys.list_peers(trust=1):
friendList.append(cls(coreInst, x)) friendList.append(cls(x))
return list(friendList) return list(friendList)

8
onionr/onionrutils/basicrequests.py
View File

@ -19,13 +19,13 @@
''' '''
import requests, streamedrequests import requests, streamedrequests
import logger, onionrexceptions import logger, onionrexceptions
def do_post_request(core_inst, url, data={}, port=0, proxyType='tor', max_size=10000): def do_post_request(onionr_inst, url, data={}, port=0, proxyType='tor', max_size=10000):
''' '''
Do a POST request through a local tor or i2p instance Do a POST request through a local tor or i2p instance
''' '''
if proxyType == 'tor': if proxyType == 'tor':
if port == 0: if port == 0:
port = core_inst.torPort port = onionr_inst.torPort
proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)} proxies = {'http': 'socks4a://127.0.0.1:' + str(port), 'https': 'socks4a://127.0.0.1:' + str(port)}
elif proxyType == 'i2p': elif proxyType == 'i2p':
proxies = {'http': 'http://127.0.0.1:4444'} proxies = {'http': 'http://127.0.0.1:4444'}
@ -44,11 +44,11 @@ def do_post_request(core_inst, url, data={}, port=0, proxyType='tor', max_size=1
retData = False retData = False
return retData return retData
def do_get_request(core_inst, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880): def do_get_request(onionr_inst, url, port=0, proxyType='tor', ignoreAPI=False, returnHeaders=False, max_size=5242880):
''' '''
Do a get request through a local tor or i2p instance Do a get request through a local tor or i2p instance
''' '''
API_VERSION = core_inst.onionrInst.API_VERSION API_VERSION = onionr_inst.onionrInst.API_VERSION
retData = False retData = False
if proxyType == 'tor': if proxyType == 'tor':
if port == 0: if port == 0:

8
onionr/onionrutils/blockmetadata.py
View File

@ -53,12 +53,12 @@ def get_block_metadata_from_data(blockData):
meta = metadata['meta'] meta = metadata['meta']
return (metadata, meta, data) return (metadata, meta, data)
def process_block_metadata(core_inst, blockHash): def process_block_metadata(blockHash):
''' '''
Read metadata from a block and cache it to the block database Read metadata from a block and cache it to the block database
''' '''
curTime = epoch.get_rounded_epoch(roundS=60) curTime = epoch.get_rounded_epoch(roundS=60)
myBlock = onionrblockapi.Block(blockHash, core_inst) myBlock = onionrblockapi.Block(blockHash)
if myBlock.isEncrypted: if myBlock.isEncrypted:
myBlock.decrypt() myBlock.decrypt()
if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted): if (myBlock.isEncrypted and myBlock.decrypted) or (not myBlock.isEncrypted):
@ -67,7 +67,7 @@ def process_block_metadata(core_inst, blockHash):
signer = bytesconverter.bytes_to_str(myBlock.signer) signer = bytesconverter.bytes_to_str(myBlock.signer)
valid = myBlock.verifySig() valid = myBlock.verifySig()
if myBlock.getMetadata('newFSKey') is not None: if myBlock.getMetadata('newFSKey') is not None:
onionrusers.OnionrUser(core_inst, signer).addForwardKey(myBlock.getMetadata('newFSKey')) onionrusers.OnionrUser(signer).addForwardKey(myBlock.getMetadata('newFSKey'))
try: try:
if len(blockType) <= 10: if len(blockType) <= 10:
@ -85,7 +85,7 @@ def process_block_metadata(core_inst, blockHash):
blockmetadb.update_block_info(blockHash, 'expire', expireTime) blockmetadb.update_block_info(blockHash, 'expire', expireTime)
if not blockType is None: if not blockType is None:
blockmetadb.update_block_info(blockHash, 'dataType', blockType) blockmetadb.update_block_info(blockHash, 'dataType', blockType)
onionrevents.event('processblocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = core_inst.onionrInst) #onionrevents.event('processblocks', data = {'block': myBlock, 'type': blockType, 'signer': signer, 'validSig': valid}, onionr = core_inst.onionrInst)
else: else:
pass pass

5
onionr/onionrutils/checkcommunicator.py
View File

@ -18,9 +18,10 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import time, os import time, os
def is_communicator_running(core_inst, timeout = 5, interval = 0.1): import filepaths
def is_communicator_running(timeout = 5, interval = 0.1):
try: try:
runcheck_file = core_inst.dataDir + '.runcheck' runcheck_file = filepaths.run_check_file
if not os.path.isfile(runcheck_file): if not os.path.isfile(runcheck_file):
open(runcheck_file, 'w+').close() open(runcheck_file, 'w+').close()

14
onionr/onionrutils/importnewblocks.py
View File

@ -18,19 +18,19 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import glob import glob
import logger, core import logger
from onionrutils import blockmetadata from onionrutils import blockmetadata
from coredb import blockmetadb from coredb import blockmetadb
def import_new_blocks(core_inst=None, scanDir=''): import filepaths, onionrcrypto
def import_new_blocks(scanDir=''):
''' '''
This function is intended to scan for new blocks ON THE DISK and import them This function is intended to scan for new blocks ON THE DISK and import them
''' '''
if core_inst is None: crypto = onionrcrypto.OnionrCrypto()
core_inst = core.Core()
blockList = blockmetadb.get_block_list() blockList = blockmetadb.get_block_list()
exist = False exist = False
if scanDir == '': if scanDir == '':
scanDir = core_inst.blockDataLocation scanDir = filepaths.block_data_location
if not scanDir.endswith('/'): if not scanDir.endswith('/'):
scanDir += '/' scanDir += '/'
for block in glob.glob(scanDir + "*.dat"): for block in glob.glob(scanDir + "*.dat"):
@ -39,10 +39,10 @@ def import_new_blocks(core_inst=None, scanDir=''):
logger.info('Found new block on dist %s' % block, terminal=True) logger.info('Found new block on dist %s' % block, terminal=True)
with open(block, 'rb') as newBlock: with open(block, 'rb') as newBlock:
block = block.replace(scanDir, '').replace('.dat', '') block = block.replace(scanDir, '').replace('.dat', '')
if core_inst._crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''): if crypto.sha3Hash(newBlock.read()) == block.replace('.dat', ''):
blockmetadb.add_to_block_DB(block.replace('.dat', ''), dataSaved=True) blockmetadb.add_to_block_DB(block.replace('.dat', ''), dataSaved=True)
logger.info('Imported block %s.' % block, terminal=True) logger.info('Imported block %s.' % block, terminal=True)
blockmetadata.process_block_metadata(core_inst, block) blockmetadata.process_block_metadata(block)
else: else:
logger.warn('Failed to verify hash for %s' % block, terminal=True) logger.warn('Failed to verify hash for %s' % block, terminal=True)
if not exist: if not exist:

5
onionr/onionrutils/mnemonickeys.py
View File

@ -19,9 +19,10 @@
''' '''
import base64 import base64
from etc import pgpwords from etc import pgpwords
def get_human_readable_ID(core_inst, pub=''): import onionrcrypto
def get_human_readable_ID(pub=''):
'''gets a human readable ID from a public key''' '''gets a human readable ID from a public key'''
if pub == '': if pub == '':
pub = core_inst._crypto.pubKey pub = onionrcrypto.OnionrCrypto().pubKey
pub = base64.b16encode(base64.b32decode(pub)).decode() pub = base64.b16encode(base64.b32decode(pub)).decode()
return ' '.join(pgpwords.wordify(pub)) return ' '.join(pgpwords.wordify(pub))

15
onionr/onionrutils/validatemetadata.py
View File

@ -21,9 +21,12 @@ import json
import logger, onionrexceptions import logger, onionrexceptions
from etc import onionrvalues from etc import onionrvalues
from onionrutils import stringvalidators, epoch, bytesconverter from onionrutils import stringvalidators, epoch, bytesconverter
def validate_metadata(core_inst, metadata, blockData): import config, onionrvalues, filepaths, onionrcrypto
def validate_metadata(metadata, blockData):
'''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string''' '''Validate metadata meets onionr spec (does not validate proof value computation), take in either dictionary or json string'''
# TODO, make this check sane sizes # TODO, make this check sane sizes
crypto = onionrcrypto.OnionrCrypto()
requirements = onionrvalues.OnionrValues()
retData = False retData = False
maxClockDifference = 120 maxClockDifference = 120
@ -35,11 +38,11 @@ def validate_metadata(core_inst, metadata, blockData):
pass pass
# Validate metadata dict for invalid keys to sizes that are too large # Validate metadata dict for invalid keys to sizes that are too large
maxAge = core_inst.config.get("general.max_block_age", onionrvalues.OnionrValues().default_expire) maxAge = config.get("general.max_block_age", onionrvalues.OnionrValues().default_expire)
if type(metadata) is dict: if type(metadata) is dict:
for i in metadata: for i in metadata:
try: try:
core_inst.requirements.blockMetadataLengths[i] requirements.blockMetadataLengths[i]
except KeyError: except KeyError:
logger.warn('Block has invalid metadata key ' + i) logger.warn('Block has invalid metadata key ' + i)
break break
@ -49,7 +52,7 @@ def validate_metadata(core_inst, metadata, blockData):
testData = len(testData) testData = len(testData)
except (TypeError, AttributeError) as e: except (TypeError, AttributeError) as e:
testData = len(str(testData)) testData = len(str(testData))
if core_inst.requirements.blockMetadataLengths[i] < testData: if requirements.blockMetadataLengths[i] < testData:
logger.warn('Block metadata key ' + i + ' exceeded maximum size') logger.warn('Block metadata key ' + i + ' exceeded maximum size')
break break
if i == 'time': if i == 'time':
@ -78,9 +81,9 @@ def validate_metadata(core_inst, metadata, blockData):
else: else:
# if metadata loop gets no errors, it does not break, therefore metadata is valid # if metadata loop gets no errors, it does not break, therefore metadata is valid
# make sure we do not have another block with the same data content (prevent data duplication and replay attacks) # make sure we do not have another block with the same data content (prevent data duplication and replay attacks)
nonce = bytesconverter.bytes_to_str(core_inst._crypto.sha3Hash(blockData)) nonce = bytesconverter.bytes_to_str(crypto.sha3Hash(blockData))
try: try:
with open(core_inst.dataNonceFile, 'r') as nonceFile: with open(filepaths.data_nonce_file, 'r') as nonceFile:
if nonce in nonceFile.read(): if nonce in nonceFile.read():
retData = False # we've seen that nonce before, so we can't pass metadata retData = False # we've seen that nonce before, so we can't pass metadata
raise onionrexceptions.DataExists raise onionrexceptions.DataExists

13
onionr/serializeddata.py
View File

@ -18,10 +18,10 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import core, json import json
from coredb import blockmetadb from coredb import blockmetadb
class SerializedData: class SerializedData:
def __init__(self, coreInst): def __init__(self, o_inst):
''' '''
Serialized data is in JSON format: Serialized data is in JSON format:
{ {
@ -30,14 +30,13 @@ class SerializedData:
etc etc
} }
''' '''
assert isinstance(coreInst, core.Core) self.o_inst = o_inst
self._core = coreInst
def getStats(self): def getStats(self):
'''Return statistics about our node''' '''Return statistics about our node'''
stats = {} stats = {}
stats['uptime'] = self._core.onionrInst.communicatorInst.getUptime() stats['uptime'] = self.o_inst.communicatorInst.getUptime()
stats['connectedNodes'] = '\n'.join(self._core.onionrInst.communicatorInst.onlinePeers) stats['connectedNodes'] = '\n'.join(self.o_inst.communicatorInst.onlinePeers)
stats['blockCount'] = len(blockmetadb.get_block_list()) stats['blockCount'] = len(blockmetadb.get_block_list())
stats['blockQueueCount'] = len(self._core.onionrInst.communicatorInst.blockQueue) stats['blockQueueCount'] = len(self.o_inst.communicatorInst.blockQueue)
return json.dumps(stats) return json.dumps(stats)

5
onionr/static-data/default-plugins/flow/main.py
View File

@ -21,7 +21,7 @@
# Imports some useful libraries # Imports some useful libraries
import threading, time, locale, sys, os import threading, time, locale, sys, os
from onionrblockapi import Block from onionrblockapi import Block
import logger, config import logger, config, onionrblocks
from onionrutils import escapeansi, epoch from onionrutils import escapeansi, epoch
locale.setlocale(locale.LC_ALL, '') locale.setlocale(locale.LC_ALL, '')
from coredb import blockmetadb from coredb import blockmetadb
@ -34,7 +34,6 @@ PLUGIN_VERSION = '0.0.1'
class OnionrFlow: class OnionrFlow:
def __init__(self): def __init__(self):
self.myCore = pluginapi.get_core()
self.alreadyOutputed = [] self.alreadyOutputed = []
self.flowRunning = False self.flowRunning = False
self.channel = None self.channel = None
@ -63,7 +62,7 @@ class OnionrFlow:
expireTime = epoch.get_epoch() + 43200 expireTime = epoch.get_epoch() + 43200
if len(message) > 0: if len(message) > 0:
logger.info('Inserting message as block...', terminal=True) logger.info('Inserting message as block...', terminal=True)
self.myCore.insertBlock(message, header='txt', expire=expireTime, meta={'ch': self.channel}) onionrblocks.insert(message, header='txt', expire=expireTime, meta={'ch': self.channel})
logger.info("Flow is exiting, goodbye", terminal=True) logger.info("Flow is exiting, goodbye", terminal=True)
return return

13
onionr/storagecounter.py
View File

@ -17,17 +17,16 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import config import config, filepaths
config.reload() config.reload()
class StorageCounter: class StorageCounter:
def __init__(self, coreInst): def __init__(self):
self._core = coreInst self.dataFile = filepaths.usage_file
self.dataFile = self._core.usageFile
return return
def isFull(self): def isFull(self):
retData = False retData = False
if self._core.config.get('allocations.disk', 2000000000) <= (self.getAmount() + 1000): if config.get('allocations.disk', 2000000000) <= (self.getAmount() + 1000):
retData = True retData = True
return retData return retData
@ -49,13 +48,13 @@ class StorageCounter:
def getPercent(self): def getPercent(self):
'''Return percent (decimal/float) of disk space we're using''' '''Return percent (decimal/float) of disk space we're using'''
amount = self.getAmount() amount = self.getAmount()
return round(amount / self._core.config.get('allocations.disk', 2000000000), 2) return round(amount / config.get('allocations.disk', 2000000000), 2)
def addBytes(self, amount): def addBytes(self, amount):
'''Record that we are now using more disk space, unless doing so would exceed configured max''' '''Record that we are now using more disk space, unless doing so would exceed configured max'''
newAmount = amount + self.getAmount() newAmount = amount + self.getAmount()
retData = newAmount retData = newAmount
if newAmount > self._core.config.get('allocations.disk', 2000000000): if newAmount > config.get('allocations.disk', 2000000000):
retData = False retData = False
else: else:
self._update(newAmount) self._update(newAmount)

18
onionr/subprocesspow.py
View File

@ -22,22 +22,18 @@
import subprocess, os import subprocess, os
import multiprocessing, threading, time, json import multiprocessing, threading, time, json
from multiprocessing import Pipe, Process from multiprocessing import Pipe, Process
import core, onionrblockapi, config, onionrutils, logger, onionrproofs import onionrblockapi, config, onionrutils, logger, onionrproofs, onionrcrypto
from onionrutils import bytesconverter from onionrutils import bytesconverter
crypto = onionrcrypto.OnionrCrypto()
class SubprocessPOW: class SubprocessPOW:
def __init__(self, data, metadata, core_inst=None, subproc_count=None): def __init__(self, data, metadata, subproc_count=None):
''' '''
Onionr proof of work using multiple processes Onionr proof of work using multiple processes
Accepts block data, block metadata Accepts block data, block metadata
and optionally an onionr core library instance.
if subproc_count is not set, os.cpu_count() is used to determine the number of processes if subproc_count is not set, os.cpu_count() is used to determine the number of processes
Do to Python GIL multiprocessing or use of external libraries is necessary to accelerate CPU bound tasks Do to Python GIL multiprocessing or use of external libraries is necessary to accelerate CPU bound tasks
''' '''
# Option to accept existing core instance to save memory
if core_inst is None:
core_inst = core.Core()
# No known benefit to using more processes than there are cores. # No known benefit to using more processes than there are cores.
# Note: os.cpu_count perhaps not always accurate # Note: os.cpu_count perhaps not always accurate
if subproc_count is None: if subproc_count is None:
@ -45,7 +41,6 @@ class SubprocessPOW:
self.subproc_count = subproc_count self.subproc_count = subproc_count
self.result = '' self.result = ''
self.shutdown = False self.shutdown = False
self.core_inst = core_inst
self.data = data self.data = data
self.metadata = metadata self.metadata = metadata
@ -54,7 +49,7 @@ class SubprocessPOW:
self.data = bytesconverter.str_to_bytes(data) self.data = bytesconverter.str_to_bytes(data)
# Calculate difficulty. Dumb for now, may use good algorithm in the future. # Calculate difficulty. Dumb for now, may use good algorithm in the future.
self.difficulty = onionrproofs.getDifficultyForNewBlock(bytes(json_metadata + b'\n' + self.data), coreInst=self.core_inst) self.difficulty = onionrproofs.getDifficultyForNewBlock(bytes(json_metadata + b'\n' + self.data)
logger.info('Computing POW (difficulty: %s)...' % self.difficulty) logger.info('Computing POW (difficulty: %s)...' % self.difficulty)
@ -101,7 +96,6 @@ class SubprocessPOW:
metadata = self.metadata metadata = self.metadata
puzzle = self.puzzle puzzle = self.puzzle
difficulty = self.difficulty difficulty = self.difficulty
mcore = core.Core() # I think we make a new core here because of multiprocess bugs
while True: while True:
# Break if shutdown received # Break if shutdown received
if pipe.poll() and pipe.recv() == 'shutdown': if pipe.poll() and pipe.recv() == 'shutdown':
@ -111,7 +105,7 @@ class SubprocessPOW:
# Serialize metadata, combine with block data # Serialize metadata, combine with block data
payload = json.dumps(metadata).encode() + b'\n' + data payload = json.dumps(metadata).encode() + b'\n' + data
# Check sha3_256 hash of block, compare to puzzle. Send payload if puzzle finished # Check sha3_256 hash of block, compare to puzzle. Send payload if puzzle finished
token = mcore._crypto.sha3Hash(payload) token = crypto.sha3Hash(payload)
token = bytesconverter.bytes_to_str(token) # ensure token is string token = bytesconverter.bytes_to_str(token) # ensure token is string
if puzzle == token[0:difficulty]: if puzzle == token[0:difficulty]:
pipe.send(payload) pipe.send(payload)

4
onionr/utils/netutils.py
View File

@ -18,7 +18,7 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
from onionrutils import basicrequests from onionrutils import basicrequests
def checkNetwork(core_inst, torPort=0): def checkNetwork(torPort=0):
'''Check if we are connected to the internet (through Tor)''' '''Check if we are connected to the internet (through Tor)'''
retData = False retData = False
connectURLs = [] connectURLs = []
@ -27,7 +27,7 @@ def checkNetwork(core_inst, torPort=0):
connectURLs = connectTest.read().split(',') connectURLs = connectTest.read().split(',')
for url in connectURLs: for url in connectURLs:
if basicrequests.do_get_request(core_inst, url, port=torPort, ignoreAPI=True) != False: if basicrequests.do_get_request(url, port=torPort, ignoreAPI=True) != False:
retData = True retData = True
break break
except FileNotFoundError: except FileNotFoundError:

16
onionr/utils/networkmerger.py
View File

@ -18,21 +18,25 @@
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
import logger import logger
def mergeAdders(newAdderList, coreInst): from coredb import keydb
import config, onionrblacklist
from utils import gettransports
def mergeAdders(newAdderList):
''' '''
Merge peer adders list to our database Merge peer adders list to our database
''' '''
blacklist = onionrblacklist.OnionrBlackList()
try: try:
retVal = False retVal = False
if newAdderList != False: if newAdderList != False:
for adder in newAdderList.split(','): for adder in newAdderList.split(','):
adder = adder.strip() adder = adder.strip()
if not adder in coreInst.listAdders(randomOrder = False) and adder != coreInst.hsAddress and not coreInst._blacklist.inBlacklist(adder): if not adder in keydb.listkeys.list_adders(randomOrder = False) and adder != gettransports.transports[0] and not blacklist.inBlacklist(adder):
if not coreInst.config.get('tor.v3onions') and len(adder) == 62: if not config.get('tor.v3onions') and len(adder) == 62:
continue continue
if coreInst.addAddress(adder): if keydb.addkeys.add_address(adder):
# Check if we have the maxmium amount of allowed stored peers # Check if we have the maximum amount of allowed stored peers
if coreInst.config.get('peers.max_stored_peers') > len(coreInst.listAdders()): if config.get('peers.max_stored_peers') > len(keydb.listkeys.list_adders()):
logger.info('Added %s to db.' % adder, timestamp = True) logger.info('Added %s to db.' % adder, timestamp = True)
retVal = True retVal = True
else: else: