renamed onionr dir and bugfixes/linting progress
This commit is contained in:
Kevin Froman
parent
2b996da17f
commit
720efe4fca
226 changed files with 179 additions and 142 deletions
3
src/onionrblocks/__init__.py
Normal file
3
src/onionrblocks/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from . import insert
|
||||
|
||||
insert = insert.insert_block
|
||||
55
src/onionrblocks/blockimporter.py
Executable file
55
src/onionrblocks/blockimporter.py
Executable file
|
|
@ -0,0 +1,55 @@
|
|||
'''
|
||||
Onionr - Private P2P Communication
|
||||
|
||||
Import block data and save it
|
||||
'''
|
||||
'''
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
import onionrexceptions, logger
|
||||
from onionrutils import validatemetadata, blockmetadata
|
||||
from coredb import blockmetadb
|
||||
from . import onionrblacklist
|
||||
import onionrstorage
|
||||
import onionrcrypto as crypto
|
||||
def importBlockFromData(content):
|
||||
blacklist = onionrblacklist.OnionrBlackList()
|
||||
retData = False
|
||||
|
||||
try:
|
||||
content = content.encode()
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
dataHash = crypto.hashers.sha3_hash(content)
|
||||
|
||||
if blacklist.inBlacklist(dataHash):
|
||||
raise onionrexceptions.BlacklistedBlock('%s is a blacklisted block' % (dataHash,))
|
||||
|
||||
metas = blockmetadata.get_block_metadata_from_data(content) # returns tuple(metadata, meta), meta is also in metadata
|
||||
metadata = metas[0]
|
||||
if validatemetadata.validate_metadata(metadata, metas[2]): # check if metadata is valid
|
||||
if crypto.cryptoutils.verify_POW(content): # check if POW is enough/correct
|
||||
logger.info('Imported block passed proof, saving.', terminal=True)
|
||||
try:
|
||||
blockHash = onionrstorage.set_data(content)
|
||||
except onionrexceptions.DiskAllocationReached:
|
||||
logger.warn('Failed to save block due to full disk allocation')
|
||||
else:
|
||||
blockmetadb.add_to_block_DB(blockHash, dataSaved=True)
|
||||
blockmetadata.process_block_metadata(blockHash) # caches block metadata values to block database
|
||||
retData = True
|
||||
else:
|
||||
raise onionrexceptions.InvalidProof
|
||||
return retData
|
||||
198
src/onionrblocks/insert.py
Normal file
198
src/onionrblocks/insert.py
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
"""
|
||||
Onionr - Private P2P Communication
|
||||
|
||||
Create and insert Onionr blocks
|
||||
"""
|
||||
"""
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
from typing import Union
|
||||
import json
|
||||
|
||||
from onionrutils import bytesconverter, epoch
|
||||
import filepaths, onionrstorage
|
||||
from . import storagecounter
|
||||
from onionrplugins import onionrevents as events
|
||||
from etc import powchoice, onionrvalues
|
||||
import config, onionrcrypto as crypto, onionrexceptions
|
||||
from onionrusers import onionrusers
|
||||
from onionrutils import localcommand, blockmetadata, stringvalidators
|
||||
import coredb
|
||||
import onionrproofs
|
||||
from onionrproofs import subprocesspow
|
||||
import logger
|
||||
from onionrtypes import UserIDSecretKey
|
||||
|
||||
def insert_block(data: Union[str, bytes], header: str ='txt',
|
||||
sign: bool =False, encryptType:str ='', symKey:str ='',
|
||||
asymPeer:str ='', meta:dict = {},
|
||||
expire:Union[int, None] =None, disableForward:bool =False,
|
||||
signing_key:UserIDSecretKey ='')->Union[str,bool]:
|
||||
"""
|
||||
Inserts a block into the network
|
||||
encryptType must be specified to encrypt a block
|
||||
"""
|
||||
our_private_key = crypto.priv_key
|
||||
our_pub_key = crypto.pub_key
|
||||
|
||||
if signing_key != '':
|
||||
# if it was specified to use an alternative private key
|
||||
our_private_key = signing_key
|
||||
our_pub_key = bytesconverter.bytes_to_str(crypto.cryptoutils.get_pub_key_from_priv(our_private_key))
|
||||
|
||||
use_subprocess = powchoice.use_subprocess(config)
|
||||
storage_counter = storagecounter.StorageCounter()
|
||||
|
||||
allocationReachedMessage = 'Cannot insert block, disk allocation reached.'
|
||||
if storage_counter.is_full():
|
||||
logger.error(allocationReachedMessage)
|
||||
raise onionrexceptions.DiskAllocationReached
|
||||
retData = False
|
||||
|
||||
if type(data) is None:
|
||||
raise ValueError('Data cannot be none')
|
||||
|
||||
createTime = epoch.get_epoch()
|
||||
|
||||
dataNonce = bytesconverter.bytes_to_str(crypto.hashers.sha3_hash(data))
|
||||
try:
|
||||
with open(filepaths.data_nonce_file, 'r') as nonces:
|
||||
if dataNonce in nonces:
|
||||
return retData
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
# record nonce
|
||||
with open(filepaths.data_nonce_file, 'a') as nonceFile:
|
||||
nonceFile.write(dataNonce + '\n')
|
||||
|
||||
#if type(data) is bytes:
|
||||
# data = data.decode()
|
||||
#data = str(data)
|
||||
plaintext = data
|
||||
plaintextMeta = {}
|
||||
plaintextPeer = asymPeer
|
||||
|
||||
retData = ''
|
||||
signature = ''
|
||||
signer = ''
|
||||
metadata = {}
|
||||
|
||||
# metadata is full block metadata, meta is internal, user specified metadata
|
||||
|
||||
# only use header if not set in provided meta
|
||||
|
||||
meta['type'] = str(header)
|
||||
|
||||
if encryptType in ('asym', 'sym'):
|
||||
metadata['encryptType'] = encryptType
|
||||
else:
|
||||
if not config.get('general.store_plaintext_blocks', True): raise onionrexceptions.InvalidMetadata("Plaintext blocks are disabled, yet a plaintext block was being inserted")
|
||||
if not encryptType in ('', None):
|
||||
raise onionrexceptions.InvalidMetadata('encryptType must be asym or sym, or blank')
|
||||
|
||||
try:
|
||||
data = data.encode()
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
if encryptType == 'asym':
|
||||
meta['rply'] = createTime # Duplicate the time in encrypted messages to prevent replays
|
||||
if not disableForward and sign and asymPeer != our_pub_key:
|
||||
try:
|
||||
forwardEncrypted = onionrusers.OnionrUser(asymPeer).forwardEncrypt(data)
|
||||
data = forwardEncrypted[0]
|
||||
meta['forwardEnc'] = True
|
||||
expire = forwardEncrypted[2] # Expire time of key. no sense keeping block after that
|
||||
except onionrexceptions.InvalidPubkey:
|
||||
pass
|
||||
#onionrusers.OnionrUser(self, asymPeer).generateForwardKey()
|
||||
fsKey = onionrusers.OnionrUser(asymPeer).generateForwardKey()
|
||||
#fsKey = onionrusers.OnionrUser(self, asymPeer).getGeneratedForwardKeys().reverse()
|
||||
meta['newFSKey'] = fsKey
|
||||
jsonMeta = json.dumps(meta)
|
||||
plaintextMeta = jsonMeta
|
||||
if sign:
|
||||
signature = crypto.signing.ed_sign(jsonMeta.encode() + data, key=our_private_key, encodeResult=True)
|
||||
signer = our_pub_key
|
||||
|
||||
if len(jsonMeta) > 1000:
|
||||
raise onionrexceptions.InvalidMetadata('meta in json encoded form must not exceed 1000 bytes')
|
||||
|
||||
# encrypt block metadata/sig/content
|
||||
if encryptType == 'sym':
|
||||
raise NotImplementedError("not yet implemented")
|
||||
elif encryptType == 'asym':
|
||||
if stringvalidators.validate_pub_key(asymPeer):
|
||||
# Encrypt block data with forward secrecy key first, but not meta
|
||||
jsonMeta = json.dumps(meta)
|
||||
jsonMeta = crypto.encryption.pub_key_encrypt(jsonMeta, asymPeer, encodedData=True).decode()
|
||||
data = crypto.encryption.pub_key_encrypt(data, asymPeer, encodedData=False)#.decode()
|
||||
signature = crypto.encryption.pub_key_encrypt(signature, asymPeer, encodedData=True).decode()
|
||||
signer = crypto.encryption.pub_key_encrypt(signer, asymPeer, encodedData=True).decode()
|
||||
try:
|
||||
onionrusers.OnionrUser(asymPeer, saveUser=True)
|
||||
except ValueError:
|
||||
# if peer is already known
|
||||
pass
|
||||
else:
|
||||
raise onionrexceptions.InvalidPubkey(asymPeer + ' is not a valid base32 encoded ed25519 key')
|
||||
|
||||
# compile metadata
|
||||
metadata['meta'] = jsonMeta
|
||||
if len(signature) > 0: # I don't like not pattern
|
||||
metadata['sig'] = signature
|
||||
metadata['signer'] = signer
|
||||
metadata['time'] = createTime
|
||||
|
||||
# ensure expire is integer and of sane length
|
||||
if type(expire) is not type(None):
|
||||
if not len(str(int(expire))) < 20: raise ValueError('expire must be valid int less than 20 digits in length')
|
||||
metadata['expire'] = expire
|
||||
|
||||
# send block data (and metadata) to POW module to get tokenized block data
|
||||
if use_subprocess:
|
||||
payload = subprocesspow.SubprocessPOW(data, metadata).start()
|
||||
else:
|
||||
payload = onionrproofs.POW(metadata, data).waitForResult()
|
||||
if payload != False:
|
||||
try:
|
||||
retData = onionrstorage.set_data(payload)
|
||||
except onionrexceptions.DiskAllocationReached:
|
||||
logger.error(allocationReachedMessage)
|
||||
retData = False
|
||||
else:
|
||||
# Tell the api server through localCommand to wait for the daemon to upload this block to make statistical analysis more difficult
|
||||
if localcommand.local_command('/ping', maxWait=10) == 'pong!':
|
||||
if config.get('general.security_level', 1) == 0:
|
||||
localcommand.local_command('/waitforshare/' + retData, post=True, maxWait=5)
|
||||
coredb.daemonqueue.daemon_queue_add('uploadBlock', retData)
|
||||
else:
|
||||
pass
|
||||
coredb.blockmetadb.add.add_to_block_DB(retData, selfInsert=True, dataSaved=True)
|
||||
|
||||
if expire is None:
|
||||
coredb.blockmetadb.update_block_info(retData, 'expire',
|
||||
createTime + onionrvalues.DEFAULT_EXPIRE)
|
||||
else:
|
||||
coredb.blockmetadb.update_block_info(retData, 'expire', expire)
|
||||
|
||||
blockmetadata.process_block_metadata(retData)
|
||||
|
||||
if retData != False:
|
||||
if plaintextPeer == onionrvalues.DENIABLE_PEER_ADDRESS:
|
||||
events.event('insertdeniable', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': bytesconverter.bytes_to_str(asymPeer)}, threaded = True)
|
||||
else:
|
||||
events.event('insertblock', {'content': plaintext, 'meta': plaintextMeta, 'hash': retData, 'peer': bytesconverter.bytes_to_str(asymPeer)}, threaded = True)
|
||||
coredb.daemonqueue.daemon_queue_add('remove_from_insert_list', data=dataNonce)
|
||||
return retData
|
||||
119
src/onionrblocks/onionrblacklist.py
Executable file
119
src/onionrblocks/onionrblacklist.py
Executable file
|
|
@ -0,0 +1,119 @@
|
|||
'''
|
||||
Onionr - Private P2P Communication
|
||||
|
||||
This file handles maintenence of a blacklist database, for blocks and peers
|
||||
'''
|
||||
'''
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
import sqlite3, os
|
||||
import logger, onionrcrypto
|
||||
from onionrutils import epoch, bytesconverter
|
||||
from coredb import dbfiles
|
||||
|
||||
class OnionrBlackList:
|
||||
def __init__(self):
|
||||
self.blacklistDB = dbfiles.blacklist_db
|
||||
|
||||
if not os.path.exists(dbfiles.blacklist_db):
|
||||
self.generateDB()
|
||||
return
|
||||
|
||||
def inBlacklist(self, data):
|
||||
hashed = bytesconverter.bytes_to_str(onionrcrypto.hashers.sha3_hash(data))
|
||||
retData = False
|
||||
|
||||
if not hashed.isalnum():
|
||||
raise Exception("Hashed data is not alpha numeric")
|
||||
if len(hashed) > 64:
|
||||
raise Exception("Hashed data is too large")
|
||||
|
||||
for i in self._dbExecute("SELECT * FROM blacklist WHERE hash = ?", (hashed,)):
|
||||
retData = True # this only executes if an entry is present by that hash
|
||||
break
|
||||
|
||||
return retData
|
||||
|
||||
def _dbExecute(self, toExec, params = ()):
|
||||
conn = sqlite3.connect(self.blacklistDB)
|
||||
c = conn.cursor()
|
||||
retData = c.execute(toExec, params)
|
||||
conn.commit()
|
||||
return retData
|
||||
|
||||
def deleteBeforeDate(self, date):
|
||||
# TODO, delete blacklist entries before date
|
||||
return
|
||||
|
||||
def deleteExpired(self, dataType=0):
|
||||
'''Delete expired entries'''
|
||||
deleteList = []
|
||||
curTime = epoch.get_epoch()
|
||||
|
||||
try:
|
||||
int(dataType)
|
||||
except AttributeError:
|
||||
raise TypeError("dataType must be int")
|
||||
|
||||
for i in self._dbExecute('SELECT * FROM blacklist WHERE dataType = ?', (dataType,)):
|
||||
if i[1] == dataType:
|
||||
if (curTime - i[2]) >= i[3]:
|
||||
deleteList.append(i[0])
|
||||
|
||||
for thing in deleteList:
|
||||
self._dbExecute("DELETE FROM blacklist WHERE hash = ?", (thing,))
|
||||
|
||||
def generateDB(self):
|
||||
return
|
||||
|
||||
def clearDB(self):
|
||||
self._dbExecute('''DELETE FROM blacklist;''')
|
||||
|
||||
def getList(self):
|
||||
data = self._dbExecute('SELECT * FROM blacklist')
|
||||
myList = []
|
||||
for i in data:
|
||||
myList.append(i[0])
|
||||
return myList
|
||||
|
||||
def addToDB(self, data, dataType=0, expire=0):
|
||||
'''Add to the blacklist. Intended to be block hash, block data, peers, or transport addresses
|
||||
0=block
|
||||
1=peer
|
||||
2=pubkey
|
||||
'''
|
||||
|
||||
# we hash the data so we can remove data entirely from our node's disk
|
||||
hashed = bytesconverter.bytes_to_str(onionrcrypto.hashers.sha3_hash(data))
|
||||
if len(hashed) > 64:
|
||||
raise Exception("Hashed data is too large")
|
||||
|
||||
if not hashed.isalnum():
|
||||
raise Exception("Hashed data is not alpha numeric")
|
||||
try:
|
||||
int(dataType)
|
||||
except ValueError:
|
||||
raise Exception("dataType is not int")
|
||||
try:
|
||||
int(expire)
|
||||
except ValueError:
|
||||
raise Exception("expire is not int")
|
||||
if self.inBlacklist(hashed):
|
||||
return
|
||||
insert = (hashed,)
|
||||
blacklistDate = epoch.get_epoch()
|
||||
try:
|
||||
self._dbExecute("INSERT INTO blacklist (hash, dataType, blacklistDate, expire) VALUES(?, ?, ?, ?);", (str(hashed), dataType, blacklistDate, expire))
|
||||
except sqlite3.IntegrityError:
|
||||
pass
|
||||
542
src/onionrblocks/onionrblockapi.py
Executable file
542
src/onionrblocks/onionrblockapi.py
Executable file
|
|
@ -0,0 +1,542 @@
|
|||
'''
|
||||
Onionr - P2P Anonymous Storage Network
|
||||
|
||||
This file contains the OnionrBlocks class which is a class for working with Onionr blocks
|
||||
'''
|
||||
'''
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
import unpaddedbase32
|
||||
import binascii
|
||||
import logger, config, onionrexceptions, nacl.exceptions
|
||||
import json, os, sys, datetime, base64, onionrstorage
|
||||
from onionrusers import onionrusers
|
||||
from onionrutils import stringvalidators, epoch
|
||||
from coredb import blockmetadb
|
||||
from onionrutils import bytesconverter
|
||||
from onionrstorage import removeblock
|
||||
import onionrblocks
|
||||
from onionrcrypto import encryption, cryptoutils as cryptoutils, signing
|
||||
class Block:
|
||||
blockCacheOrder = list() # NEVER write your own code that writes to this!
|
||||
blockCache = dict() # should never be accessed directly, look at Block.getCache()
|
||||
|
||||
def __init__(self, hash = None, type = None, content = None, expire=None, decrypt=False, bypassReplayCheck=False):
|
||||
# take from arguments
|
||||
# sometimes people input a bytes object instead of str in `hash`
|
||||
if (not hash is None) and isinstance(hash, bytes):
|
||||
hash = hash.decode()
|
||||
|
||||
self.hash = hash
|
||||
self.btype = type
|
||||
self.bcontent = content
|
||||
self.expire = expire
|
||||
self.bypassReplayCheck = bypassReplayCheck
|
||||
|
||||
# initialize variables
|
||||
self.valid = True
|
||||
self.raw = None
|
||||
self.signed = False
|
||||
self.signature = None
|
||||
self.signedData = None
|
||||
self.blockFile = None
|
||||
self.bheader = {}
|
||||
self.bmetadata = {}
|
||||
self.isEncrypted = False
|
||||
self.decrypted = False
|
||||
self.signer = None
|
||||
self.validSig = False
|
||||
self.autoDecrypt = decrypt
|
||||
|
||||
self.update()
|
||||
|
||||
def decrypt(self, encodedData = True):
|
||||
'''
|
||||
Decrypt a block, loading decrypted data into their vars
|
||||
'''
|
||||
|
||||
if self.decrypted:
|
||||
return True
|
||||
retData = False
|
||||
# decrypt data
|
||||
if self.getHeader('encryptType') == 'asym':
|
||||
try:
|
||||
try:
|
||||
self.bcontent = encryption.pub_key_decrypt(self.bcontent, encodedData=encodedData)
|
||||
except (binascii.Error, ValueError) as e:
|
||||
self.bcontent = encryption.pub_key_decrypt(self.bcontent, encodedData=False)
|
||||
bmeta = encryption.pub_key_decrypt(self.bmetadata, encodedData=encodedData)
|
||||
try:
|
||||
bmeta = bmeta.decode()
|
||||
except AttributeError:
|
||||
# yet another bytes fix
|
||||
pass
|
||||
self.bmetadata = json.loads(bmeta)
|
||||
self.signature = encryption.pub_key_decrypt(self.signature, encodedData=encodedData)
|
||||
self.signer = encryption.pub_key_decrypt(self.signer, encodedData=encodedData)
|
||||
self.bheader['signer'] = self.signer.decode()
|
||||
self.signedData = json.dumps(self.bmetadata).encode() + self.bcontent
|
||||
|
||||
if not self.signer is None:
|
||||
if not self.verifySig():
|
||||
raise onionrexceptions.SignatureError("Block has invalid signature")
|
||||
|
||||
# Check for replay attacks
|
||||
try:
|
||||
if epoch.get_epoch() - blockmetadb.get_block_date(self.hash) > 60:
|
||||
if not cryptoutils.replay_validator(self.bmetadata['rply']): raise onionrexceptions.ReplayAttack
|
||||
except (AssertionError, KeyError, TypeError, onionrexceptions.ReplayAttack) as e:
|
||||
if not self.bypassReplayCheck:
|
||||
# Zero out variables to prevent reading of replays
|
||||
self.bmetadata = {}
|
||||
self.signer = ''
|
||||
self.bheader['signer'] = ''
|
||||
self.signedData = ''
|
||||
self.signature = ''
|
||||
raise onionrexceptions.ReplayAttack('Signature is too old. possible replay attack')
|
||||
try:
|
||||
if not self.bmetadata['forwardEnc']: raise KeyError
|
||||
except (AssertionError, KeyError) as e:
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
self.bcontent = onionrusers.OnionrUser(self.signer).forwardDecrypt(self.bcontent)
|
||||
except (onionrexceptions.DecryptionError, nacl.exceptions.CryptoError) as e:
|
||||
#logger.error(str(e))
|
||||
pass
|
||||
except nacl.exceptions.CryptoError:
|
||||
pass
|
||||
#logger.debug('Could not decrypt block. Either invalid key or corrupted data')
|
||||
except onionrexceptions.ReplayAttack:
|
||||
logger.warn('%s is possibly a replay attack' % (self.hash,))
|
||||
else:
|
||||
retData = True
|
||||
self.decrypted = True
|
||||
return retData
|
||||
|
||||
def verifySig(self):
|
||||
'''
|
||||
Verify if a block's signature is signed by its claimed signer
|
||||
'''
|
||||
if self.signer is None:
|
||||
return False
|
||||
if signing.ed_verify(data=self.signedData, key=self.signer, sig=self.signature, encodedData=True):
|
||||
self.validSig = True
|
||||
else:
|
||||
self.validSig = False
|
||||
return self.validSig
|
||||
|
||||
def update(self, data = None, file = None):
|
||||
'''
|
||||
Loads data from a block in to the current object.
|
||||
|
||||
Inputs:
|
||||
- data (str):
|
||||
- if None: will load from file by hash
|
||||
- else: will load from `data` string
|
||||
- file (str):
|
||||
- if None: will load from file specified in this parameter
|
||||
- else: will load from wherever block is stored by hash
|
||||
|
||||
Outputs:
|
||||
- (bool): indicates whether or not the operation was successful
|
||||
'''
|
||||
try:
|
||||
# import from string
|
||||
blockdata = data
|
||||
|
||||
# import from file
|
||||
if blockdata is None:
|
||||
try:
|
||||
blockdata = onionrstorage.getData(self.getHash())#.decode()
|
||||
except AttributeError:
|
||||
raise onionrexceptions.NoDataAvailable('Block does not exist')
|
||||
else:
|
||||
self.blockFile = None
|
||||
# parse block
|
||||
self.raw = blockdata
|
||||
self.bheader = json.loads(self.getRaw()[:self.getRaw().index(b'\n')])
|
||||
self.bcontent = self.getRaw()[self.getRaw().index(b'\n') + 1:]
|
||||
if ('encryptType' in self.bheader) and (self.bheader['encryptType'] in ('asym', 'sym')):
|
||||
self.bmetadata = self.getHeader('meta', None)
|
||||
self.isEncrypted = True
|
||||
else:
|
||||
self.bmetadata = json.loads(self.getHeader('meta', None))
|
||||
self.btype = self.getMetadata('type', None)
|
||||
self.signed = ('sig' in self.getHeader() and self.getHeader('sig') != '')
|
||||
# TODO: detect if signer is hash of pubkey or not
|
||||
self.signer = self.getHeader('signer', None)
|
||||
self.signature = self.getHeader('sig', None)
|
||||
# signed data is jsonMeta + block content (no linebreak)
|
||||
self.signedData = (None if not self.isSigned() else self.getHeader('meta').encode() + self.getContent())
|
||||
self.date = blockmetadb.get_block_date(self.getHash())
|
||||
self.claimedTime = self.getHeader('time', None)
|
||||
|
||||
if not self.getDate() is None:
|
||||
self.date = datetime.datetime.fromtimestamp(self.getDate())
|
||||
|
||||
self.valid = True
|
||||
|
||||
if self.autoDecrypt:
|
||||
self.decrypt()
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warn('Failed to parse block %s' % self.getHash(), error = e, timestamp = False)
|
||||
|
||||
# if block can't be parsed, it's a waste of precious space. Throw it away.
|
||||
if not self.delete():
|
||||
logger.warn('Failed to delete invalid block %s.' % self.getHash(), error = e)
|
||||
else:
|
||||
logger.debug('Deleted invalid block %s.' % self.getHash(), timestamp = False)
|
||||
|
||||
self.valid = False
|
||||
return False
|
||||
|
||||
def delete(self):
|
||||
'''
|
||||
Deletes the block's file and records, if they exist
|
||||
|
||||
Outputs:
|
||||
- (bool): whether or not the operation was successful
|
||||
'''
|
||||
|
||||
if self.exists():
|
||||
try:
|
||||
os.remove(self.getBlockFile())
|
||||
except TypeError:
|
||||
pass
|
||||
b_hash = self.getHash()
|
||||
onionrstorage.deleteBlock(b_hash)
|
||||
removeblock.remove_block(b_hash)
|
||||
return True
|
||||
return False
|
||||
|
||||
def save(self, sign = False, recreate = True):
|
||||
'''
|
||||
Saves a block to file and imports it into Onionr
|
||||
|
||||
Inputs:
|
||||
- sign (bool): whether or not to sign the block before saving
|
||||
- recreate (bool): if the block already exists, whether or not to recreate the block and save under a new hash
|
||||
|
||||
Outputs:
|
||||
- (bool): whether or not the operation was successful
|
||||
'''
|
||||
|
||||
try:
|
||||
if self.isValid() is True:
|
||||
|
||||
self.hash = onionrblocks.insert(self.getRaw(), header = self.getType(), sign = sign, meta = self.getMetadata(), expire = self.getExpire())
|
||||
if self.hash != False:
|
||||
self.update()
|
||||
|
||||
return self.getHash()
|
||||
else:
|
||||
logger.warn('Not writing block; it is invalid.')
|
||||
except Exception as e:
|
||||
logger.error('Failed to save block.', error = e, timestamp = False)
|
||||
|
||||
return False
|
||||
|
||||
# getters
|
||||
|
||||
def getExpire(self):
|
||||
'''
|
||||
Returns the expire time for a block
|
||||
|
||||
Outputs:
|
||||
- (int): the expire time for a block, or None
|
||||
'''
|
||||
return self.expire
|
||||
|
||||
def getHash(self):
|
||||
'''
|
||||
Returns the hash of the block if saved to file
|
||||
|
||||
Outputs:
|
||||
- (str): the hash of the block, or None
|
||||
'''
|
||||
|
||||
return self.hash
|
||||
|
||||
def getType(self):
|
||||
'''
|
||||
Returns the type of the block
|
||||
|
||||
Outputs:
|
||||
- (str): the type of the block
|
||||
'''
|
||||
return self.btype
|
||||
|
||||
def getRaw(self):
|
||||
'''
|
||||
Returns the raw contents of the block, if saved to file
|
||||
|
||||
Outputs:
|
||||
- (bytes): the raw contents of the block, or None
|
||||
'''
|
||||
|
||||
return self.raw
|
||||
|
||||
def getHeader(self, key = None, default = None):
|
||||
'''
|
||||
Returns the header information
|
||||
|
||||
Inputs:
|
||||
- key (str): only returns the value of the key in the header
|
||||
|
||||
Outputs:
|
||||
- (dict/str): either the whole header as a dict, or one value
|
||||
'''
|
||||
|
||||
if not key is None:
|
||||
if key in self.getHeader():
|
||||
return self.getHeader()[key]
|
||||
return default
|
||||
return self.bheader
|
||||
|
||||
def getMetadata(self, key = None, default = None):
|
||||
'''
|
||||
Returns the metadata information
|
||||
|
||||
Inputs:
|
||||
- key (str): only returns the value of the key in the metadata
|
||||
|
||||
Outputs:
|
||||
- (dict/str): either the whole metadata as a dict, or one value
|
||||
'''
|
||||
|
||||
if not key is None:
|
||||
if key in self.getMetadata():
|
||||
return self.getMetadata()[key]
|
||||
return default
|
||||
return self.bmetadata
|
||||
|
||||
def getContent(self):
|
||||
'''
|
||||
Returns the contents of the block
|
||||
|
||||
Outputs:
|
||||
- (str): the contents of the block
|
||||
'''
|
||||
|
||||
return self.bcontent
|
||||
|
||||
def getDate(self):
|
||||
'''
|
||||
Returns the date that the block was received, if loaded from file
|
||||
|
||||
Outputs:
|
||||
- (datetime): the date that the block was received
|
||||
'''
|
||||
|
||||
return self.date
|
||||
|
||||
def getBlockFile(self):
|
||||
'''
|
||||
Returns the location of the block file if it is saved
|
||||
|
||||
Outputs:
|
||||
- (str): the location of the block file, or None
|
||||
'''
|
||||
|
||||
return self.blockFile
|
||||
|
||||
def isValid(self):
|
||||
'''
|
||||
Checks if the block is valid
|
||||
|
||||
Outputs:
|
||||
- (bool): whether or not the block is valid
|
||||
'''
|
||||
|
||||
return self.valid
|
||||
|
||||
def isSigned(self):
|
||||
'''
|
||||
Checks if the block was signed
|
||||
|
||||
Outputs:
|
||||
- (bool): whether or not the block is signed
|
||||
'''
|
||||
|
||||
return self.signed
|
||||
|
||||
def getSignature(self):
|
||||
'''
|
||||
Returns the base64-encoded signature
|
||||
|
||||
Outputs:
|
||||
- (str): the signature, or None
|
||||
'''
|
||||
|
||||
return self.signature
|
||||
|
||||
def getSignedData(self):
|
||||
'''
|
||||
Returns the data that was signed
|
||||
|
||||
Outputs:
|
||||
- (str): the data that was signed, or None
|
||||
'''
|
||||
|
||||
return self.signedData
|
||||
|
||||
def isSigner(self, signer, encodedData = True):
|
||||
'''
|
||||
Checks if the block was signed by the signer inputted
|
||||
|
||||
Inputs:
|
||||
- signer (str): the public key of the signer to check against
|
||||
- encodedData (bool): whether or not the `signer` argument is base64 encoded
|
||||
|
||||
Outputs:
|
||||
- (bool): whether or not the signer of the block is the signer inputted
|
||||
'''
|
||||
signer = unpaddedbase32.repad(bytesconverter.str_to_bytes(signer))
|
||||
try:
|
||||
if (not self.isSigned()) or (not stringvalidators.validate_pub_key(signer)):
|
||||
return False
|
||||
|
||||
return bool(signing.ed_verify(self.getSignedData(), signer, self.getSignature(), encodedData = encodedData))
|
||||
except:
|
||||
return False
|
||||
|
||||
# setters
|
||||
|
||||
def setType(self, btype):
|
||||
'''
|
||||
Sets the type of the block
|
||||
|
||||
Inputs:
|
||||
- btype (str): the type of block to be set to
|
||||
|
||||
Outputs:
|
||||
- (Block): the Block instance
|
||||
'''
|
||||
|
||||
self.btype = btype
|
||||
return self
|
||||
|
||||
def setMetadata(self, key, val):
|
||||
'''
|
||||
Sets a custom metadata value
|
||||
|
||||
Metadata should not store block-specific data structures.
|
||||
|
||||
Inputs:
|
||||
- key (str): the key
|
||||
- val: the value (type is irrelevant)
|
||||
|
||||
Outputs:
|
||||
- (Block): the Block instance
|
||||
'''
|
||||
|
||||
self.bmetadata[key] = val
|
||||
return self
|
||||
|
||||
def setContent(self, bcontent):
|
||||
'''
|
||||
Sets the contents of the block
|
||||
|
||||
Inputs:
|
||||
- bcontent (str): the contents to be set to
|
||||
|
||||
Outputs:
|
||||
- (Block): the Block instance
|
||||
'''
|
||||
|
||||
self.bcontent = str(bcontent)
|
||||
return self
|
||||
|
||||
# static functions
|
||||
|
||||
def getBlocks(type = None, signer = None, signed = None, reverse = False, limit = None):
|
||||
'''
|
||||
Returns a list of Block objects based on supplied filters
|
||||
|
||||
Inputs:
|
||||
- type (str): filters by block type
|
||||
- signer (str/list): filters by signer (one in the list has to be a signer)
|
||||
- signed (bool): filters out by whether or not the block is signed
|
||||
- reverse (bool): reverses the list if True
|
||||
|
||||
Outputs:
|
||||
- (list): a list of Block objects that match the input
|
||||
'''
|
||||
|
||||
try:
|
||||
|
||||
relevant_blocks = list()
|
||||
blocks = (blockmetadb.get_block_list() if type is None else blockmetadb.get_blocks_by_type(type))
|
||||
|
||||
for block in blocks:
|
||||
if Block.exists(block):
|
||||
block = Block(block)
|
||||
|
||||
relevant = True
|
||||
|
||||
if (not signed is None) and (block.isSigned() != bool(signed)):
|
||||
relevant = False
|
||||
if not signer is None:
|
||||
if isinstance(signer, (str,)):
|
||||
signer = [signer]
|
||||
if isinstance(signer, (bytes,)):
|
||||
signer = [signer.decode()]
|
||||
|
||||
isSigner = False
|
||||
for key in signer:
|
||||
if block.isSigner(key):
|
||||
isSigner = True
|
||||
break
|
||||
|
||||
if not isSigner:
|
||||
relevant = False
|
||||
|
||||
if relevant and (limit is None or len(relevant_Blocks) <= int(limit)):
|
||||
relevant_blocks.append(block)
|
||||
|
||||
if bool(reverse):
|
||||
relevant_blocks.reverse()
|
||||
|
||||
return relevant_blocks
|
||||
except Exception as e:
|
||||
logger.debug('Failed to get blocks.', error = e)
|
||||
|
||||
return list()
|
||||
|
||||
def exists(bHash):
|
||||
'''
|
||||
Checks if a block is saved to file or not
|
||||
|
||||
Inputs:
|
||||
- hash (str/Block):
|
||||
- if (Block): check if this block is saved to file
|
||||
- if (str): check if a block by this hash is in file
|
||||
|
||||
Outputs:
|
||||
- (bool): whether or not the block file exists
|
||||
'''
|
||||
|
||||
# no input data? scrap it.
|
||||
if bHash is None:
|
||||
return False
|
||||
|
||||
if isinstance(bHash, Block):
|
||||
bHash = bHash.getHash()
|
||||
|
||||
ret = isinstance(onionrstorage.getData(bHash), type(None))
|
||||
|
||||
return not ret
|
||||
69
src/onionrblocks/storagecounter.py
Executable file
69
src/onionrblocks/storagecounter.py
Executable file
|
|
@ -0,0 +1,69 @@
|
|||
"""
|
||||
Onionr - Private P2P Communication
|
||||
|
||||
Keeps track of how much disk space we're using
|
||||
"""
|
||||
"""
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
import config, filepaths
|
||||
config.reload()
|
||||
class StorageCounter:
|
||||
def __init__(self):
|
||||
self.data_file = filepaths.usage_file
|
||||
return
|
||||
|
||||
def is_full(self)->bool:
|
||||
"""Returns if the allocated disk space is full (this is Onionr config, not true FS capacity)"""
|
||||
ret_data = False
|
||||
if config.get('allocations.disk', 2000000000) <= (self.get_amount() + 1000):
|
||||
ret_data = True
|
||||
return ret_data
|
||||
|
||||
def _update(self, data):
|
||||
with open(self.data_file, 'w') as data_file:
|
||||
data_file.write(str(data))
|
||||
|
||||
def get_amount(self)->int:
|
||||
"""Return how much disk space we're using (according to record)"""
|
||||
ret_data = 0
|
||||
try:
|
||||
with open(self.data_file, 'r') as data_file:
|
||||
ret_data = int(data_file.read())
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except ValueError:
|
||||
pass # Possibly happens when the file is empty
|
||||
return ret_data
|
||||
|
||||
def get_percent(self)->int:
|
||||
"""Return percent (decimal/float) of disk space we're using"""
|
||||
amount = self.get_amount()
|
||||
return round(amount / config.get('allocations.disk', 2000000000), 2)
|
||||
|
||||
def add_bytes(self, amount)->int:
|
||||
"""Record that we are now using more disk space, unless doing so would exceed configured max"""
|
||||
new_amount = amount + self.get_amount()
|
||||
ret_data = new_amount
|
||||
if new_amount > config.get('allocations.disk', 2000000000):
|
||||
ret_data = False
|
||||
else:
|
||||
self._update(new_amount)
|
||||
return ret_data
|
||||
|
||||
def remove_bytes(self, amount)->int:
|
||||
"""Record that we are now using less disk space"""
|
||||
new_amount = self.get_amount() - amount
|
||||
self._update(new_amount)
|
||||
return new_amount
|
||||
Loading…
Add table
Add a link
Reference in a new issue