renamed communicator, bug fixes and added work on onionfragment.py
parent
0f4626a68c
commit
e60503771e
|
@ -567,6 +567,7 @@ class OnionrCommunicatorDaemon:
|
||||||
# when inserting a block, we try to upload it to a few peers to add some deniability
|
# when inserting a block, we try to upload it to a few peers to add some deniability
|
||||||
triedPeers = []
|
triedPeers = []
|
||||||
finishedUploads = []
|
finishedUploads = []
|
||||||
|
self.blocksToUpload = self._core._crypto.randomShuffle(self.blocksToUpload)
|
||||||
if len(self.blocksToUpload) != 0:
|
if len(self.blocksToUpload) != 0:
|
||||||
for bl in self.blocksToUpload:
|
for bl in self.blocksToUpload:
|
||||||
if not self._core._utils.validateHash(bl):
|
if not self._core._utils.validateHash(bl):
|
|
@ -99,6 +99,7 @@ class Core:
|
||||||
logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation)
|
logger.warn('Warning: address bootstrap file not found ' + self.bootstrapFileLocation)
|
||||||
|
|
||||||
self._utils = onionrutils.OnionrUtils(self)
|
self._utils = onionrutils.OnionrUtils(self)
|
||||||
|
self.blockCache = onionrstorage.BlockCache()
|
||||||
# Initialize the crypto object
|
# Initialize the crypto object
|
||||||
self._crypto = onionrcrypto.OnionrCrypto(self)
|
self._crypto = onionrcrypto.OnionrCrypto(self)
|
||||||
self._blacklist = onionrblacklist.OnionrBlackList(self)
|
self._blacklist = onionrblacklist.OnionrBlackList(self)
|
||||||
|
|
|
@ -33,7 +33,7 @@ import onionrutils
|
||||||
import netcontroller
|
import netcontroller
|
||||||
from netcontroller import NetController
|
from netcontroller import NetController
|
||||||
from onionrblockapi import Block
|
from onionrblockapi import Block
|
||||||
import onionrproofs, onionrexceptions, onionrusers, communicator2
|
import onionrproofs, onionrexceptions, onionrusers, communicator
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from urllib3.contrib.socks import SOCKSProxyManager
|
from urllib3.contrib.socks import SOCKSProxyManager
|
||||||
|
@ -710,7 +710,6 @@ class Onionr:
|
||||||
'''
|
'''
|
||||||
Starts the Onionr communication daemon
|
Starts the Onionr communication daemon
|
||||||
'''
|
'''
|
||||||
communicatorDaemon = './communicator2.py'
|
|
||||||
|
|
||||||
# remove runcheck if it exists
|
# remove runcheck if it exists
|
||||||
if os.path.isfile('data/.runcheck'):
|
if os.path.isfile('data/.runcheck'):
|
||||||
|
@ -750,10 +749,8 @@ class Onionr:
|
||||||
logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey))
|
logger.debug('Using public key: %s' % (logger.colors.underline + self.onionrCore._crypto.pubKey))
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
|
|
||||||
# TODO: make runable on windows
|
|
||||||
#communicatorProc = subprocess.Popen([communicatorDaemon, 'run', str(net.socksPort)])
|
|
||||||
self.onionrCore.torPort = net.socksPort
|
self.onionrCore.torPort = net.socksPort
|
||||||
communicatorThread = Thread(target=communicator2.startCommunicator, args=(self, str(net.socksPort)))
|
communicatorThread = Thread(target=communicator.startCommunicator, args=(self, str(net.socksPort)))
|
||||||
communicatorThread.start()
|
communicatorThread.start()
|
||||||
|
|
||||||
while self.communicatorInst is None:
|
while self.communicatorInst is None:
|
||||||
|
@ -940,7 +937,8 @@ class Onionr:
|
||||||
logger.error('Block hash is invalid')
|
logger.error('Block hash is invalid')
|
||||||
return
|
return
|
||||||
|
|
||||||
Block.mergeChain(bHash, fileName)
|
with open(fileName, 'wb') as myFile:
|
||||||
|
myFile.write(base64.b64decode(Block(bHash, core=self.onionrCore).bcontent))
|
||||||
return
|
return
|
||||||
|
|
||||||
def addWebpage(self):
|
def addWebpage(self):
|
||||||
|
@ -963,12 +961,9 @@ class Onionr:
|
||||||
return
|
return
|
||||||
logger.info('Adding file... this might take a long time.')
|
logger.info('Adding file... this might take a long time.')
|
||||||
try:
|
try:
|
||||||
if singleBlock:
|
with open(filename, 'rb') as singleFile:
|
||||||
with open(filename, 'rb') as singleFile:
|
blockhash = self.onionrCore.insertBlock(base64.b64encode(singleFile.read()), header=blockType)
|
||||||
blockhash = self.onionrCore.insertBlock(base64.b64encode(singleFile.read()), header=blockType)
|
logger.info('File %s saved in block %s' % (filename, blockhash))
|
||||||
else:
|
|
||||||
blockhash = Block.createChain(file = filename)
|
|
||||||
logger.info('File %s saved in block %s.' % (filename, blockhash))
|
|
||||||
except:
|
except:
|
||||||
logger.error('Failed to save file in block.', timestamp = False)
|
logger.error('Failed to save file in block.', timestamp = False)
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
'''
|
'''
|
||||||
Onionr - P2P Anonymous Storage Network
|
Onionr - P2P Anonymous Storage Network
|
||||||
|
|
||||||
This class contains the OnionrBlocks class which is a class for working with Onionr blocks
|
This file contains the OnionrBlocks class which is a class for working with Onionr blocks
|
||||||
'''
|
'''
|
||||||
'''
|
'''
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
|
@ -56,18 +56,7 @@ class Block:
|
||||||
if self.getCore() is None:
|
if self.getCore() is None:
|
||||||
self.core = onionrcore.Core()
|
self.core = onionrcore.Core()
|
||||||
|
|
||||||
# update the blocks' contents if it exists
|
self.update()
|
||||||
if not self.getHash() is None:
|
|
||||||
if not self.core._utils.validateHash(self.hash):
|
|
||||||
logger.debug('Block hash %s is invalid.' % self.getHash())
|
|
||||||
raise onionrexceptions.InvalidHexHash('Block hash is invalid.')
|
|
||||||
elif not self.update():
|
|
||||||
logger.debug('Failed to open block %s.' % self.getHash())
|
|
||||||
else:
|
|
||||||
pass
|
|
||||||
#logger.debug('Did not update block.')
|
|
||||||
|
|
||||||
# logic
|
|
||||||
|
|
||||||
def decrypt(self, anonymous = True, encodedData = True):
|
def decrypt(self, anonymous = True, encodedData = True):
|
||||||
'''
|
'''
|
||||||
|
@ -140,13 +129,15 @@ class Block:
|
||||||
Outputs:
|
Outputs:
|
||||||
- (bool): indicates whether or not the operation was successful
|
- (bool): indicates whether or not the operation was successful
|
||||||
'''
|
'''
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# import from string
|
# import from string
|
||||||
blockdata = data
|
blockdata = data
|
||||||
|
|
||||||
# import from file
|
# import from file
|
||||||
if blockdata is None:
|
if blockdata is None:
|
||||||
|
blockdata = onionrstorage.getData(self.core, self.getHash()).decode()
|
||||||
|
'''
|
||||||
|
|
||||||
filelocation = file
|
filelocation = file
|
||||||
|
|
||||||
readfile = True
|
readfile = True
|
||||||
|
@ -169,6 +160,7 @@ class Block:
|
||||||
#blockdata = f.read().decode()
|
#blockdata = f.read().decode()
|
||||||
|
|
||||||
self.blockFile = filelocation
|
self.blockFile = filelocation
|
||||||
|
'''
|
||||||
else:
|
else:
|
||||||
self.blockFile = None
|
self.blockFile = None
|
||||||
# parse block
|
# parse block
|
||||||
|
@ -588,7 +580,7 @@ class Block:
|
||||||
|
|
||||||
return list()
|
return list()
|
||||||
|
|
||||||
def mergeChain(child, file = None, maximumFollows = 32, core = None):
|
def mergeChain(child, file = None, maximumFollows = 1000, core = None):
|
||||||
'''
|
'''
|
||||||
Follows a child Block to its root parent Block, merging content
|
Follows a child Block to its root parent Block, merging content
|
||||||
|
|
||||||
|
@ -635,7 +627,7 @@ class Block:
|
||||||
|
|
||||||
blocks.append(block.getHash())
|
blocks.append(block.getHash())
|
||||||
|
|
||||||
buffer = ''
|
buffer = b''
|
||||||
|
|
||||||
# combine block contents
|
# combine block contents
|
||||||
for hash in blocks:
|
for hash in blocks:
|
||||||
|
@ -644,101 +636,17 @@ class Block:
|
||||||
contents = base64.b64decode(contents.encode())
|
contents = base64.b64decode(contents.encode())
|
||||||
|
|
||||||
if file is None:
|
if file is None:
|
||||||
buffer += contents.decode()
|
try:
|
||||||
|
buffer += contents.encode()
|
||||||
|
except AttributeError:
|
||||||
|
buffer += contents
|
||||||
else:
|
else:
|
||||||
file.write(contents)
|
file.write(contents)
|
||||||
file.close()
|
if file is not None:
|
||||||
|
file.close()
|
||||||
|
|
||||||
return (None if not file is None else buffer)
|
return (None if not file is None else buffer)
|
||||||
|
|
||||||
def createChain(data = None, chunksize = 99800, file = None, type = 'chunk', sign = True, encrypt = False, verbose = False):
|
|
||||||
'''
|
|
||||||
Creates a chain of blocks to store larger amounts of data
|
|
||||||
|
|
||||||
The chunksize is set to 99800 because it provides the least amount of PoW for the most amount of data.
|
|
||||||
|
|
||||||
Inputs:
|
|
||||||
- data (*): if `file` is None, the data to be stored in blocks
|
|
||||||
- file (file/str): the filename or file object to read from (or None to read `data` instead)
|
|
||||||
- chunksize (int): the number of bytes per block chunk
|
|
||||||
- type (str): the type header for each of the blocks
|
|
||||||
- sign (bool): whether or not to sign each block
|
|
||||||
- encrypt (str): the public key to encrypt to, or False to disable encryption
|
|
||||||
- verbose (bool): whether or not to return a tuple containing more info
|
|
||||||
|
|
||||||
Outputs:
|
|
||||||
- if `verbose`:
|
|
||||||
- (tuple):
|
|
||||||
- (str): the child block hash
|
|
||||||
- (list): all block hashes associated with storing the file
|
|
||||||
- if not `verbose`:
|
|
||||||
- (str): the child block hash
|
|
||||||
'''
|
|
||||||
|
|
||||||
blocks = list()
|
|
||||||
|
|
||||||
# initial datatype checks
|
|
||||||
if data is None and file is None:
|
|
||||||
return blocks
|
|
||||||
elif not (file is None or (isinstance(file, str) and os.path.exists(file))):
|
|
||||||
return blocks
|
|
||||||
elif isinstance(file, str):
|
|
||||||
file = open(file, 'rb')
|
|
||||||
if not isinstance(data, str):
|
|
||||||
data = str(data)
|
|
||||||
|
|
||||||
if not file is None:
|
|
||||||
filesize = os.stat(file.name).st_size
|
|
||||||
offset = filesize % chunksize
|
|
||||||
maxtimes = int(filesize / chunksize)
|
|
||||||
|
|
||||||
for times in range(0, maxtimes + 1):
|
|
||||||
# read chunksize bytes from the file (end -> beginning)
|
|
||||||
if times < maxtimes:
|
|
||||||
file.seek(- ((times + 1) * chunksize), 2)
|
|
||||||
content = file.read(chunksize)
|
|
||||||
else:
|
|
||||||
file.seek(0, 0)
|
|
||||||
content = file.read(offset)
|
|
||||||
|
|
||||||
# encode it- python is really bad at handling certain bytes that
|
|
||||||
# are often present in binaries.
|
|
||||||
content = base64.b64encode(content).decode()
|
|
||||||
|
|
||||||
# if it is the end of the file, exit
|
|
||||||
if not content:
|
|
||||||
break
|
|
||||||
|
|
||||||
# create block
|
|
||||||
block = Block()
|
|
||||||
block.setType(type)
|
|
||||||
block.setContent(content)
|
|
||||||
block.setParent((blocks[-1] if len(blocks) != 0 else None))
|
|
||||||
hash = block.save(sign = sign)
|
|
||||||
|
|
||||||
# remember the hash in cache
|
|
||||||
blocks.append(hash)
|
|
||||||
elif not data is None:
|
|
||||||
for content in reversed([data[n:n + chunksize] for n in range(0, len(data), chunksize)]):
|
|
||||||
# encode chunk with base64
|
|
||||||
content = base64.b64encode(content.encode()).decode()
|
|
||||||
|
|
||||||
# create block
|
|
||||||
block = Block()
|
|
||||||
block.setType(type)
|
|
||||||
block.setContent(content)
|
|
||||||
block.setParent((blocks[-1] if len(blocks) != 0 else None))
|
|
||||||
hash = block.save(sign = sign)
|
|
||||||
|
|
||||||
# remember the hash in cache
|
|
||||||
blocks.append(hash)
|
|
||||||
|
|
||||||
# return different things depending on verbosity
|
|
||||||
if verbose:
|
|
||||||
return (blocks[-1], blocks)
|
|
||||||
file.close()
|
|
||||||
return blocks[-1]
|
|
||||||
|
|
||||||
def exists(bHash):
|
def exists(bHash):
|
||||||
'''
|
'''
|
||||||
Checks if a block is saved to file or not
|
Checks if a block is saved to file or not
|
||||||
|
@ -799,7 +707,7 @@ class Block:
|
||||||
if block.getHash() in Block.getCache() and not override:
|
if block.getHash() in Block.getCache() and not override:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# dump old cached blocks if the size exeeds the maximum
|
# dump old cached blocks if the size exceeds the maximum
|
||||||
if sys.getsizeof(Block.blockCacheOrder) >= config.get('allocations.block_cache_total', 50000000): # 50MB default cache size
|
if sys.getsizeof(Block.blockCacheOrder) >= config.get('allocations.block_cache_total', 50000000): # 50MB default cache size
|
||||||
del Block.blockCache[blockCacheOrder.pop(0)]
|
del Block.blockCache[blockCacheOrder.pop(0)]
|
||||||
|
|
||||||
|
|
|
@ -34,7 +34,7 @@ class DaemonTools:
|
||||||
'''Announce our node to our peers'''
|
'''Announce our node to our peers'''
|
||||||
retData = False
|
retData = False
|
||||||
announceFail = False
|
announceFail = False
|
||||||
if config.get('general.security_level') == 0:
|
if self.daemon._core.config('general.security_level') == 0:
|
||||||
# Announce to random online peers
|
# Announce to random online peers
|
||||||
for i in self.daemon.onlinePeers:
|
for i in self.daemon.onlinePeers:
|
||||||
if not i in self.announceCache:
|
if not i in self.announceCache:
|
||||||
|
|
|
@ -53,6 +53,9 @@ class BlacklistedBlock(Exception):
|
||||||
class DataExists(Exception):
|
class DataExists(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
class NoDataAvailable(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
class InvalidHexHash(Exception):
|
class InvalidHexHash(Exception):
|
||||||
'''When a string is not a valid hex string of appropriate length for a hash value'''
|
'''When a string is not a valid hex string of appropriate length for a hash value'''
|
||||||
pass
|
pass
|
||||||
|
|
|
@ -0,0 +1,73 @@
|
||||||
|
'''
|
||||||
|
Onionr - P2P Anonymous Storage Network
|
||||||
|
|
||||||
|
This file contains the OnionrFragment class which implements the fragment system
|
||||||
|
'''
|
||||||
|
'''
|
||||||
|
This program is free software: you can redistribute it and/or modify
|
||||||
|
it under the terms of the GNU General Public License as published by
|
||||||
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
(at your option) any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License
|
||||||
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
'''
|
||||||
|
|
||||||
|
# onionr:10ch+10ch+10chgdecryptionkey
|
||||||
|
import core, sys, binascii, os
|
||||||
|
|
||||||
|
FRAGMENT_SIZE = 0.25
|
||||||
|
TRUNCATE_LENGTH = 30
|
||||||
|
|
||||||
|
class OnionrFragment:
|
||||||
|
def __init__(self, uri=None):
|
||||||
|
uri = uri.replace('onionr:', '')
|
||||||
|
count = 0
|
||||||
|
blocks = []
|
||||||
|
appendData = ''
|
||||||
|
key = ''
|
||||||
|
for x in uri:
|
||||||
|
if x == 'k':
|
||||||
|
key = uri[uri.index('k') + 1:]
|
||||||
|
appendData += x
|
||||||
|
if count == TRUNCATE_LENGTH:
|
||||||
|
blocks.append(appendData)
|
||||||
|
appendData = ''
|
||||||
|
count = 0
|
||||||
|
count += 1
|
||||||
|
self.key = key
|
||||||
|
self.blocks = blocks
|
||||||
|
return
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def generateFragments(data=None, coreInst=None):
|
||||||
|
if coreInst is None:
|
||||||
|
coreInst = core.Core()
|
||||||
|
|
||||||
|
key = os.urandom(32)
|
||||||
|
data = coreInst._crypto.symmetricEncrypt(data, key).decode()
|
||||||
|
blocks = []
|
||||||
|
blockData = b""
|
||||||
|
uri = "onionr:"
|
||||||
|
total = sys.getsizeof(data)
|
||||||
|
for x in data:
|
||||||
|
blockData += x.encode()
|
||||||
|
if round(len(blockData) / len(data), 3) > FRAGMENT_SIZE:
|
||||||
|
blocks.append(core.Core().insertBlock(blockData))
|
||||||
|
blockData = b""
|
||||||
|
|
||||||
|
for bl in blocks:
|
||||||
|
uri += bl[:TRUNCATE_LENGTH]
|
||||||
|
uri += "k"
|
||||||
|
uri += binascii.hexlify(key).decode()
|
||||||
|
return (uri, key)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
uri = OnionrFragment.generateFragments("test")[0]
|
||||||
|
print(uri)
|
||||||
|
OnionrFragment(uri)
|
|
@ -21,6 +21,13 @@ import core, sys, sqlite3, os, dbcreator
|
||||||
|
|
||||||
DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
|
DB_ENTRY_SIZE_LIMIT = 10000 # Will be a config option
|
||||||
|
|
||||||
|
class BlockCache:
|
||||||
|
def __init__(self):
|
||||||
|
self.blocks = {}
|
||||||
|
def cleanCache(self):
|
||||||
|
while sys.getsizeof(self.blocks) > 100000000:
|
||||||
|
self.blocks.pop(list(self.blocks.keys())[0])
|
||||||
|
|
||||||
def dbCreate(coreInst):
|
def dbCreate(coreInst):
|
||||||
try:
|
try:
|
||||||
dbcreator.DBCreator(coreInst).createBlockDataDB()
|
dbcreator.DBCreator(coreInst).createBlockDataDB()
|
||||||
|
@ -62,6 +69,7 @@ def store(coreInst, data, blockHash=''):
|
||||||
else:
|
else:
|
||||||
with open('%s/%s.dat' % (coreInst.blockDataLocation, blockHash), 'wb') as blockFile:
|
with open('%s/%s.dat' % (coreInst.blockDataLocation, blockHash), 'wb') as blockFile:
|
||||||
blockFile.write(data)
|
blockFile.write(data)
|
||||||
|
coreInst.blockCache.cleanCache()
|
||||||
|
|
||||||
def getData(coreInst, bHash):
|
def getData(coreInst, bHash):
|
||||||
assert isinstance(coreInst, core.Core)
|
assert isinstance(coreInst, core.Core)
|
||||||
|
|
Loading…
Reference in New Issue