Add createChain and mergeChain
parent
6ff3c2c519
commit
9c2acb7099
|
@ -193,7 +193,7 @@ class Onionr:
|
||||||
'add-addr': self.addAddress,
|
'add-addr': self.addAddress,
|
||||||
'addaddr': self.addAddress,
|
'addaddr': self.addAddress,
|
||||||
'addaddress': self.addAddress,
|
'addaddress': self.addAddress,
|
||||||
|
|
||||||
'add-file': self.addFile,
|
'add-file': self.addFile,
|
||||||
'addfile': self.addFile,
|
'addfile': self.addFile,
|
||||||
|
|
||||||
|
@ -467,7 +467,12 @@ class Onionr:
|
||||||
|
|
||||||
os.makedirs(plugins.get_plugins_folder(plugin_name))
|
os.makedirs(plugins.get_plugins_folder(plugin_name))
|
||||||
with open(plugins.get_plugins_folder(plugin_name) + '/main.py', 'a') as main:
|
with open(plugins.get_plugins_folder(plugin_name) + '/main.py', 'a') as main:
|
||||||
main.write(open('static-data/default_plugin.py').read().replace('$user', os.getlogin()).replace('$date', datetime.datetime.now().strftime('%Y-%m-%d')).replace('$name', plugin_name))
|
contents = ''
|
||||||
|
with open('static-data/default_plugin.py', 'rb') as file:
|
||||||
|
contents = file.read()
|
||||||
|
|
||||||
|
# TODO: Fix $user. os.getlogin() is B U G G Y
|
||||||
|
main.write(contents.replace('$user', 'some random developer').replace('$date', datetime.datetime.now().strftime('%Y-%m-%d')).replace('$name', plugin_name))
|
||||||
|
|
||||||
with open(plugins.get_plugins_folder(plugin_name) + '/info.json', 'a') as main:
|
with open(plugins.get_plugins_folder(plugin_name) + '/info.json', 'a') as main:
|
||||||
main.write(json.dumps({'author' : 'anonymous', 'description' : 'the default description of the plugin', 'version' : '1.0'}))
|
main.write(json.dumps({'author' : 'anonymous', 'description' : 'the default description of the plugin', 'version' : '1.0'}))
|
||||||
|
@ -567,7 +572,7 @@ class Onionr:
|
||||||
# define stats messages here
|
# define stats messages here
|
||||||
totalBlocks = len(Block.getBlocks())
|
totalBlocks = len(Block.getBlocks())
|
||||||
signedBlocks = len(Block.getBlocks(signed = True))
|
signedBlocks = len(Block.getBlocks(signed = True))
|
||||||
|
|
||||||
messages = {
|
messages = {
|
||||||
# info about local client
|
# info about local client
|
||||||
'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 2) else logger.colors.fg.red + 'Offline'),
|
'Onionr Daemon Status' : ((logger.colors.fg.green + 'Online') if self.onionrUtils.isCommunicatorRunning(timeout = 2) else logger.colors.fg.red + 'Offline'),
|
||||||
|
@ -585,7 +590,7 @@ class Onionr:
|
||||||
'Known Peers Count' : str(len(self.onionrCore.listPeers()) - 1),
|
'Known Peers Count' : str(len(self.onionrCore.listPeers()) - 1),
|
||||||
'Enabled Plugins Count' : str(len(config.get('plugins')['enabled'])) + ' / ' + str(len(os.listdir('data/plugins/'))),
|
'Enabled Plugins Count' : str(len(config.get('plugins')['enabled'])) + ' / ' + str(len(os.listdir('data/plugins/'))),
|
||||||
'Known Blocks Count' : str(totalBlocks),
|
'Known Blocks Count' : str(totalBlocks),
|
||||||
'Percent Blocks Signed' : str(round(100 * signedBlocks / totalBlocks, 2)) + '%'
|
'Percent Blocks Signed' : str(round(100 * signedBlocks / totalBlocks, 2)) + '%' # TODO: div by zero error
|
||||||
}
|
}
|
||||||
|
|
||||||
# color configuration
|
# color configuration
|
||||||
|
@ -654,16 +659,16 @@ class Onionr:
|
||||||
if len(sys.argv) >= 3:
|
if len(sys.argv) >= 3:
|
||||||
filename = sys.argv[2]
|
filename = sys.argv[2]
|
||||||
contents = None
|
contents = None
|
||||||
|
|
||||||
if not os.path.exists(filename):
|
if not os.path.exists(filename):
|
||||||
logger.warn('That file does not exist. Improper path?')
|
logger.warn('That file does not exist. Improper path?')
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(filename, 'rb') as file:
|
with open(filename, 'rb') as file:
|
||||||
contents = file.read().decode()
|
contents = file.read().decode()
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if not contents is None:
|
if not contents is None:
|
||||||
blockhash = Block('bin', contents).save()
|
blockhash = Block('bin', contents).save()
|
||||||
logger.info('File %s saved in block %s.' % (filename, blockhash))
|
logger.info('File %s saved in block %s.' % (filename, blockhash))
|
||||||
|
|
|
@ -19,7 +19,7 @@
|
||||||
'''
|
'''
|
||||||
|
|
||||||
import core as onionrcore, logger
|
import core as onionrcore, logger
|
||||||
import json, os, datetime
|
import json, os, datetime, base64
|
||||||
|
|
||||||
class Block:
|
class Block:
|
||||||
def __init__(self, hash = None, core = None):
|
def __init__(self, hash = None, core = None):
|
||||||
|
@ -254,15 +254,15 @@ class Block:
|
||||||
'''
|
'''
|
||||||
|
|
||||||
return str(self.bcontent)
|
return str(self.bcontent)
|
||||||
|
|
||||||
def getParent(self):
|
def getParent(self):
|
||||||
'''
|
'''
|
||||||
Returns the Block's parent Block, or None
|
Returns the Block's parent Block, or None
|
||||||
|
|
||||||
Outputs:
|
Outputs:
|
||||||
- (Block): the Block's parent
|
- (Block): the Block's parent
|
||||||
'''
|
'''
|
||||||
|
|
||||||
return self.parent
|
return self.parent
|
||||||
|
|
||||||
def getDate(self):
|
def getDate(self):
|
||||||
|
@ -360,21 +360,21 @@ class Block:
|
||||||
|
|
||||||
self.btype = btype
|
self.btype = btype
|
||||||
return self
|
return self
|
||||||
|
|
||||||
def setMetadata(self, key, val):
|
def setMetadata(self, key, val):
|
||||||
'''
|
'''
|
||||||
Sets a custom metadata value
|
Sets a custom metadata value
|
||||||
|
|
||||||
Metadata should not store block-specific data structures.
|
Metadata should not store block-specific data structures.
|
||||||
|
|
||||||
Inputs:
|
Inputs:
|
||||||
- key (str): the key
|
- key (str): the key
|
||||||
- val: the value (type is irrelevant)
|
- val: the value (type is irrelevant)
|
||||||
|
|
||||||
Outputs:
|
Outputs:
|
||||||
- (Block): the Block instance
|
- (Block): the Block instance
|
||||||
'''
|
'''
|
||||||
|
|
||||||
if key == 'parent' and (not val is None) and (not val == self.getParent().getHash()):
|
if key == 'parent' and (not val is None) and (not val == self.getParent().getHash()):
|
||||||
self.setParent(val)
|
self.setParent(val)
|
||||||
else:
|
else:
|
||||||
|
@ -394,21 +394,21 @@ class Block:
|
||||||
|
|
||||||
self.bcontent = str(bcontent)
|
self.bcontent = str(bcontent)
|
||||||
return self
|
return self
|
||||||
|
|
||||||
def setParent(self, parent):
|
def setParent(self, parent):
|
||||||
'''
|
'''
|
||||||
Sets the Block's parent
|
Sets the Block's parent
|
||||||
|
|
||||||
Inputs:
|
Inputs:
|
||||||
- parent (Block/str): the Block's parent, to be stored in metadata
|
- parent (Block/str): the Block's parent, to be stored in metadata
|
||||||
|
|
||||||
Outputs:
|
Outputs:
|
||||||
- (Block): the Block instance
|
- (Block): the Block instance
|
||||||
'''
|
'''
|
||||||
|
|
||||||
if type(parent) == str:
|
if type(parent) == str:
|
||||||
parent = Block(parent, core = self.getCore())
|
parent = Block(parent, core = self.getCore())
|
||||||
|
|
||||||
self.parent = parent
|
self.parent = parent
|
||||||
self.setMetadata('parent', (None if parent is None else self.getParent().getHash()))
|
self.setMetadata('parent', (None if parent is None else self.getParent().getHash()))
|
||||||
return self
|
return self
|
||||||
|
@ -469,77 +469,95 @@ class Block:
|
||||||
|
|
||||||
return list()
|
return list()
|
||||||
|
|
||||||
def merge(child, file = None, maximumFollows = 32, core = None):
|
def mergeChain(child, file = None, maximumFollows = 32, core = None):
|
||||||
'''
|
'''
|
||||||
Follows a child Block to its root parent Block, merging content
|
Follows a child Block to its root parent Block, merging content
|
||||||
|
|
||||||
Inputs:
|
Inputs:
|
||||||
- child (str/Block): the child Block to be followed
|
- child (str/Block): the child Block to be followed
|
||||||
- file (str/file): the file to write the content to, instead of returning it
|
- file (str/file): the file to write the content to, instead of returning it
|
||||||
- maximumFollows (int): the maximum number of Blocks to follow
|
- maximumFollows (int): the maximum number of Blocks to follow
|
||||||
|
|
||||||
'''
|
'''
|
||||||
|
|
||||||
# validate data and instantiate Core
|
# validate data and instantiate Core
|
||||||
core = (core if not core is None else onionrcore.Core())
|
core = (core if not core is None else onionrcore.Core())
|
||||||
maximumFollows = max(0, maximumFollows)
|
maximumFollows = max(0, maximumFollows)
|
||||||
|
|
||||||
# type conversions
|
# type conversions
|
||||||
|
if type(child) == list:
|
||||||
|
child = child[-1]
|
||||||
if type(child) == str:
|
if type(child) == str:
|
||||||
child = Block(child)
|
child = Block(child)
|
||||||
if (not file is None) and (type(file) == str):
|
if (not file is None) and (type(file) == str):
|
||||||
file = open(file, 'ab')
|
file = open(file, 'ab')
|
||||||
|
|
||||||
# only store hashes to avoid intensive memory usage
|
# only store hashes to avoid intensive memory usage
|
||||||
blocks = [child.getHash()]
|
blocks = [child.getHash()]
|
||||||
|
|
||||||
# generate a list of parent Blocks
|
# generate a list of parent Blocks
|
||||||
while True:
|
while True:
|
||||||
# end if the maximum number of follows has been exceeded
|
# end if the maximum number of follows has been exceeded
|
||||||
if len(blocks) - 1 >= maximumFollows:
|
if len(blocks) - 1 >= maximumFollows:
|
||||||
break
|
break
|
||||||
|
|
||||||
block = Block(blocks[-1], core = core).getParent()
|
block = Block(blocks[-1], core = core).getParent()
|
||||||
|
|
||||||
# end if there is no parent Block
|
# end if there is no parent Block
|
||||||
if block is None:
|
if block is None:
|
||||||
break
|
break
|
||||||
|
|
||||||
# end if the Block is pointing to a previously parsed Block
|
# end if the Block is pointing to a previously parsed Block
|
||||||
if block.getHash() in blocks:
|
if block.getHash() in blocks:
|
||||||
break
|
break
|
||||||
|
|
||||||
# end if the block is not valid
|
# end if the block is not valid
|
||||||
if not block.isValid():
|
if not block.isValid():
|
||||||
break
|
break
|
||||||
|
|
||||||
blocks.append(block.getHash())
|
blocks.append(block.getHash())
|
||||||
|
|
||||||
buffer = ''
|
buffer = ''
|
||||||
|
|
||||||
# combine block contents
|
# combine block contents
|
||||||
for hash in blocks:
|
for hash in blocks:
|
||||||
block = Block(hash, core = core)
|
block = Block(hash, core = core)
|
||||||
contents = block.getContent()
|
contents = block.getContent()
|
||||||
|
contents = base64.b64decode(contents.encode())
|
||||||
|
|
||||||
if file is None:
|
if file is None:
|
||||||
buffer += contents
|
buffer += contents.decode()
|
||||||
else:
|
else:
|
||||||
file.write(contents)
|
file.write(contents)
|
||||||
|
|
||||||
return (None if not file is None else buffer)
|
return (None if not file is None else buffer)
|
||||||
|
|
||||||
def create(data = None, chunksize = 4999000, file = None, type = 'chunk', sign = True):
|
def createChain(data = None, chunksize = 99800, file = None, type = 'chunk', sign = True, encrypt = False, verbose = False):
|
||||||
'''
|
'''
|
||||||
Creates a chain of blocks to store larger amounts of data
|
Creates a chain of blocks to store larger amounts of data
|
||||||
|
|
||||||
The chunksize is set to 4999000 because it provides the least amount of PoW for the most amount of data.
|
The chunksize is set to 99800 because it provides the least amount of PoW for the most amount of data.
|
||||||
|
|
||||||
TODO: Add docs
|
Inputs:
|
||||||
|
- data (*): if `file` is None, the data to be stored in blocks
|
||||||
|
- file (file/str): the filename or file object to read from (or None to read `data` instead)
|
||||||
|
- chunksize (int): the number of bytes per block chunk
|
||||||
|
- type (str): the type header for each of the blocks
|
||||||
|
- sign (bool): whether or not to sign each block
|
||||||
|
- encrypt (str): the public key to encrypt to, or False to disable encryption
|
||||||
|
- verbose (bool): whether or not to return a tuple containing more info
|
||||||
|
|
||||||
|
Outputs:
|
||||||
|
- if `verbose`:
|
||||||
|
- (tuple):
|
||||||
|
- (str): the child block hash
|
||||||
|
- (list): all block hashes associated with storing the file
|
||||||
|
- if not `verbose`:
|
||||||
|
- (str): the child block hash
|
||||||
'''
|
'''
|
||||||
|
|
||||||
blocks = list()
|
blocks = list()
|
||||||
|
|
||||||
# initial datatype checks
|
# initial datatype checks
|
||||||
if data is None and file is None:
|
if data is None and file is None:
|
||||||
return blocks
|
return blocks
|
||||||
|
@ -547,41 +565,60 @@ class Block:
|
||||||
return blocks
|
return blocks
|
||||||
elif isinstance(file, str):
|
elif isinstance(file, str):
|
||||||
file = open(file, 'rb')
|
file = open(file, 'rb')
|
||||||
if isinstance(data, str):
|
if not isinstance(data, str):
|
||||||
data = str(data)
|
data = str(data)
|
||||||
|
|
||||||
if not file is None:
|
if not file is None:
|
||||||
while True:
|
filesize = os.stat(file.name).st_size
|
||||||
# read chunksize bytes from the file
|
offset = filesize % chunksize
|
||||||
content = file.read(chunksize)
|
maxtimes = int(filesize / chunksize)
|
||||||
|
|
||||||
|
for times in range(0, maxtimes + 1):
|
||||||
|
# read chunksize bytes from the file (end -> beginning)
|
||||||
|
if times < maxtimes:
|
||||||
|
file.seek(- ((times + 1) * chunksize), 2)
|
||||||
|
content = file.read(chunksize)
|
||||||
|
else:
|
||||||
|
file.seek(0, 0)
|
||||||
|
content = file.read(offset)
|
||||||
|
|
||||||
|
# encode it- python is really bad at handling certain bytes that
|
||||||
|
# are often present in binaries.
|
||||||
|
content = base64.b64encode(content).decode()
|
||||||
|
|
||||||
# if it is the end of the file, exit
|
# if it is the end of the file, exit
|
||||||
if not content:
|
if not content:
|
||||||
break
|
break
|
||||||
|
|
||||||
# create block
|
# create block
|
||||||
block = Block()
|
block = Block()
|
||||||
block.setType(type)
|
block.setType(type)
|
||||||
block.setContent(content)
|
block.setContent(content)
|
||||||
block.setParent((blocks[-1] if len(blocks) != 0 else None))
|
block.setParent((blocks[-1] if len(blocks) != 0 else None))
|
||||||
hash = block.save(sign = sign)
|
hash = block.save(sign = sign)
|
||||||
|
|
||||||
# remember the hash in cache
|
# remember the hash in cache
|
||||||
blocks.append(hash)
|
blocks.append(hash)
|
||||||
elif not data is None:
|
elif not data is None:
|
||||||
for content in [data[n:n + chunksize] for n in range(0, len(data), chunksize)]:
|
for content in reversed([data[n:n + chunksize] for n in range(0, len(data), chunksize)]):
|
||||||
|
# encode chunk with base64
|
||||||
|
content = base64.b64encode(content.encode()).decode()
|
||||||
|
|
||||||
# create block
|
# create block
|
||||||
block = Block()
|
block = Block()
|
||||||
block.setType(type)
|
block.setType(type)
|
||||||
block.setContent(content)
|
block.setContent(content)
|
||||||
block.setParent((blocks[-1] if len(blocks) != 0 else None))
|
block.setParent((blocks[-1] if len(blocks) != 0 else None))
|
||||||
hash = block.save(sign = sign)
|
hash = block.save(sign = sign)
|
||||||
|
|
||||||
# remember the hash in cache
|
# remember the hash in cache
|
||||||
blocks.append(hash)
|
blocks.append(hash)
|
||||||
|
|
||||||
return blocks
|
# return different things depending on verbosity
|
||||||
|
if verbose:
|
||||||
|
return (blocks[-1], blocks)
|
||||||
|
return blocks[-1]
|
||||||
|
|
||||||
def exists(hash):
|
def exists(hash):
|
||||||
'''
|
'''
|
||||||
Checks if a block is saved to file or not
|
Checks if a block is saved to file or not
|
||||||
|
|
|
@ -133,30 +133,23 @@ class OnionrTests(unittest.TestCase):
|
||||||
if not block.getContent() == content:
|
if not block.getContent() == content:
|
||||||
logger.warn('Test block content is invalid! (%s != %s)' % (block.getContent(), content))
|
logger.warn('Test block content is invalid! (%s != %s)' % (block.getContent(), content))
|
||||||
self.assertTrue(False)
|
self.assertTrue(False)
|
||||||
|
|
||||||
logger.debug('-'*26 + '\n')
|
logger.debug('-'*26 + '\n')
|
||||||
logger.info('Running BlockAPI test #2...')
|
logger.info('Running BlockAPI test #2...')
|
||||||
|
|
||||||
original_content = 'onionr'
|
|
||||||
contents = [original_content[i:i+2] for i in range(0, len(original_content), 2)]
|
|
||||||
contents.reverse()
|
|
||||||
|
|
||||||
blocks = list()
|
|
||||||
parent = None
|
|
||||||
|
|
||||||
for content in contents:
|
|
||||||
block = Block('test', content)
|
|
||||||
block.setParent(parent)
|
|
||||||
parent = block
|
|
||||||
print('block "%s": %s' % (content, block.save()))
|
|
||||||
blocks.append(block)
|
|
||||||
|
|
||||||
child = blocks[-1]
|
original_content = 'onionr'
|
||||||
|
|
||||||
merged = Block.merge(child)
|
logger.debug('original: %s' % original_content)
|
||||||
|
|
||||||
print('merged blocks: %s' % merged)
|
blocks = Block.createChain(data = original_content, chunksize = 2, verbose = True)
|
||||||
|
|
||||||
|
logger.debug(blocks[1])
|
||||||
|
|
||||||
|
child = blocks[0]
|
||||||
|
merged = Block.mergeChain(child)
|
||||||
|
|
||||||
|
logger.debug('merged blocks (child: %s): %s' % (child, merged))
|
||||||
|
|
||||||
if merged != original_content:
|
if merged != original_content:
|
||||||
self.assertTrue(False)
|
self.assertTrue(False)
|
||||||
self.assertTrue(True)
|
self.assertTrue(True)
|
||||||
|
|
Loading…
Reference in New Issue