renamed onionr dir and bugfixes/linting progress

This commit is contained in:
Kevin Froman 2019-11-20 04:52:50 -06:00
parent 2b996da17f
commit 720efe4fca
226 changed files with 179 additions and 142 deletions

1
src/coredb/__init__.py Normal file
View file

@ -0,0 +1 @@
from . import keydb, blockmetadb, daemonqueue

View file

@ -0,0 +1,83 @@
'''
Onionr - Private P2P Communication
This module works with information relating to blocks stored on the node
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
from etc import onionrvalues
from . import expiredblocks, updateblockinfo, add
from .. import dbfiles
update_block_info = updateblockinfo.update_block_info
add_to_block_DB = add.add_to_block_DB
def get_block_list(dateRec = None, unsaved = False):
'''
Get list of our blocks
'''
if dateRec == None:
dateRec = 0
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
execute = 'SELECT hash FROM hashes WHERE dateReceived >= ? ORDER BY dateReceived ASC;'
args = (dateRec,)
rows = list()
for row in c.execute(execute, args):
for i in row:
rows.append(i)
conn.close()
return rows
def get_block_date(blockHash):
'''
Returns the date a block was received
'''
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
execute = 'SELECT dateReceived FROM hashes WHERE hash=?;'
args = (blockHash,)
for row in c.execute(execute, args):
for i in row:
return int(i)
conn.close()
return None
def get_blocks_by_type(blockType, orderDate=True):
'''
Returns a list of blocks by the type
'''
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
if orderDate:
execute = 'SELECT hash FROM hashes WHERE dataType=? ORDER BY dateReceived;'
else:
execute = 'SELECT hash FROM hashes WHERE dataType=?;'
args = (blockType,)
rows = list()
for row in c.execute(execute, args):
for i in row:
rows.append(i)
conn.close()
return rows

View file

@ -0,0 +1,43 @@
'''
Onionr - Private P2P Communication
Add an entry to the block metadata database
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import os, sqlite3, secrets
from onionrutils import epoch, blockmetadata
from etc import onionrvalues
from .. import dbfiles
def add_to_block_DB(newHash, selfInsert=False, dataSaved=False):
'''
Add a hash value to the block db
Should be in hex format!
'''
if blockmetadata.has_block(newHash):
return
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
currentTime = epoch.get_epoch() + secrets.randbelow(301)
if selfInsert or dataSaved:
selfInsert = 1
else:
selfInsert = 0
data = (newHash, currentTime, '', selfInsert)
c.execute('INSERT INTO hashes (hash, dateReceived, dataType, dataSaved) VALUES(?, ?, ?, ?);', data)
conn.commit()
conn.close()

View file

@ -0,0 +1,39 @@
'''
Onionr - Private P2P Communication
Get a list of expired blocks still stored
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
from onionrutils import epoch
from .. import dbfiles
from etc import onionrvalues
def get_expired_blocks():
'''Returns a list of expired blocks'''
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
date = int(epoch.get_epoch())
compiled = (date,)
execute = 'SELECT hash FROM hashes WHERE expire <= ? ORDER BY dateReceived;'
rows = list()
for row in c.execute(execute, compiled):
for i in row:
rows.append(i)
conn.close()
return rows

View file

@ -0,0 +1,50 @@
'''
Onionr - Private P2P Communication
Update block information in the metadata database by a field name
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
from .. import dbfiles
from etc import onionrvalues
def update_block_info(hash, key, data):
'''
sets info associated with a block
hash - the hash of a block
dateReceived - the date the block was recieved, not necessarily when it was created
decrypted - if we can successfully decrypt the block (does not describe its current state)
dataType - data type of the block
dataFound - if the data has been found for the block
dataSaved - if the data has been saved for the block
sig - optional signature by the author (not optional if author is specified)
author - multi-round partial sha3-256 hash of authors public key
dateClaimed - timestamp claimed inside the block, only as trustworthy as the block author is
expire - expire date for a block
'''
if key not in ('dateReceived', 'decrypted', 'dataType', 'dataFound',
'dataSaved', 'sig', 'author', 'dateClaimed', 'expire'):
raise ValueError('Key must be in the allowed list')
conn = sqlite3.connect(dbfiles.block_meta_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
args = (data, hash)
# Unfortunately, not really possible
c.execute("UPDATE hashes SET " + key + " = ? where hash = ?;", args)
conn.commit()
conn.close()
return True

View file

@ -0,0 +1,93 @@
'''
Onionr - Private P2P Communication
Write and read the daemon queue, which is how messages are passed into the onionr daemon in a more
direct way than the http api
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3, os
from onionrplugins import onionrevents as events
from onionrutils import localcommand, epoch
from .. import dbfiles
from onionrsetup import dbcreator
from etc import onionrvalues
def daemon_queue()->str:
'''
Gives commands to the communication proccess/daemon by reading an sqlite3 database
This function intended to be used by the client. Queue to exchange data between "client" and server.
'''
retData = False
if not os.path.exists(dbfiles.daemon_queue_db):
dbcreator.createDaemonDB()
else:
conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
try:
for row in c.execute('SELECT command, data, date, min(ID), responseID FROM commands group by id'):
retData = row
break
except sqlite3.OperationalError:
dbcreator.createDaemonDB()
else:
if retData != False:
c.execute('DELETE FROM commands WHERE id=?;', (retData[3],))
conn.commit()
conn.close()
return retData
def daemon_queue_add(command: str, data='', responseID: str =''):
'''
Add a command to the daemon queue, used by the communication daemon (communicator.py)
'''
retData = True
date = epoch.get_epoch()
conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
t = (command, data, date, responseID)
try:
c.execute('INSERT INTO commands (command, data, date, responseID) VALUES(?, ?, ?, ?)', t)
conn.commit()
except sqlite3.OperationalError:
retData = False
daemon_queue()
conn.close()
return retData
def daemon_queue_get_response(responseID=''):
'''
Get a response sent by communicator to the API, by requesting to the API
'''
if len(responseID) == 0: raise ValueError('ResponseID should not be empty')
resp = localcommand.local_command(dbfiles.daemon_queue_db, 'queueResponse/' + responseID)
return resp
def clear_daemon_queue():
'''
Clear the daemon queue (somewhat dangerous)
'''
conn = sqlite3.connect(dbfiles.daemon_queue_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
c.execute('DELETE FROM commands;')
conn.commit()
conn.close()

12
src/coredb/dbfiles.py Normal file
View file

@ -0,0 +1,12 @@
from utils import identifyhome
import filepaths
home = identifyhome.identify_home()
if not home.endswith('/'): home += '/'
block_meta_db = '%sblock-metadata.db' % (home)
block_data_db = '%s/block-data.db' % (filepaths.block_data_location,)
daemon_queue_db = '%sdaemon-queue.db' % (home,)
address_info_db = '%saddress.db' % (home,)
user_id_info_db = '%susers.db' % (home,)
forward_keys_db = '%sforward-keys.db' % (home,)
blacklist_db = '%sblacklist.db' % (home,)

View file

@ -0,0 +1 @@
from . import addkeys, listkeys, removekeys, userinfo, transportinfo

View file

@ -0,0 +1,96 @@
'''
Onionr - Private P2P Communication
add user keys or transport addresses
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
from onionrplugins import onionrevents as events
from onionrutils import stringvalidators
from . import listkeys
from utils import gettransports
from .. import dbfiles
import onionrcrypto
from etc import onionrvalues
def add_peer(peerID, name=''):
'''
Adds a public key to the key database (misleading function name)
'''
if peerID in listkeys.list_peers() or peerID == onionrcrypto.pub_key:
raise ValueError("specified id is already known")
# This function simply adds a peer to the DB
if not stringvalidators.validate_pub_key(peerID):
return False
#events.event('pubkey_add', data = {'key': peerID}, onionr = core_inst.onionrInst)
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
hashID = ""
c = conn.cursor()
t = (peerID, name, 'unknown', hashID, 0)
for i in c.execute("SELECT * FROM peers WHERE id = ?;", (peerID,)):
try:
if i[0] == peerID:
conn.close()
return False
except ValueError:
pass
except IndexError:
pass
c.execute('INSERT INTO peers (id, name, dateSeen, hashID, trust) VALUES(?, ?, ?, ?, ?);', t)
conn.commit()
conn.close()
return True
def add_address(address):
'''
Add an address to the address database (only tor currently)
'''
if type(address) is None or len(address) == 0:
return False
if stringvalidators.validate_transport(address):
if address in gettransports.get():
return False
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
# check if address is in database
# this is safe to do because the address is validated above, but we strip some chars here too just in case
address = address.replace('\'', '').replace(';', '').replace('"', '').replace('\\', '')
for i in c.execute("SELECT * FROM adders WHERE address = ?;", (address,)):
try:
if i[0] == address:
conn.close()
return False
except ValueError:
pass
except IndexError:
pass
t = (address, 1)
c.execute('INSERT INTO adders (address, type) VALUES(?, ?);', t)
conn.commit()
conn.close()
#events.event('address_add', data = {'address': address}, onionr = core_inst.onionrInst)
return True
else:
return False

View file

@ -0,0 +1,86 @@
'''
Onionr - Private P2P Communication
get lists for user keys or transport addresses
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
import logger
from onionrutils import epoch
from etc import onionrvalues
from .. import dbfiles
from . import userinfo, transportinfo
def list_peers(randomOrder=True, getPow=False, trust=0):
'''
Return a list of public keys (misleading function name)
randomOrder determines if the list should be in a random order
trust sets the minimum trust to list
'''
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
payload = ''
if trust not in (0, 1, 2):
logger.error('Tried to select invalid trust.')
return
if randomOrder:
payload = 'SELECT * FROM peers WHERE trust >= ? ORDER BY RANDOM();'
else:
payload = 'SELECT * FROM peers WHERE trust >= ?;'
peerList = []
for i in c.execute(payload, (trust,)):
try:
if len(i[0]) != 0:
if getPow:
peerList.append(i[0] + '-' + i[1])
else:
peerList.append(i[0])
except TypeError:
pass
conn.close()
return peerList
def list_adders(randomOrder=True, i2p=True, recent=0):
'''
Return a list of transport addresses
'''
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
if randomOrder:
addresses = c.execute('SELECT * FROM adders ORDER BY RANDOM();')
else:
addresses = c.execute('SELECT * FROM adders;')
addressList = []
for i in addresses:
if len(i[0].strip()) == 0:
continue
addressList.append(i[0])
conn.close()
testList = list(addressList) # create new list to iterate
for address in testList:
try:
if recent > 0 and (epoch.get_epoch() - transportinfo.get_address_info(address, 'lastConnect')) > recent:
raise TypeError # If there is no last-connected date or it was too long ago, don't add peer to list if recent is not 0
except TypeError:
addressList.remove(address)
return addressList

View file

@ -0,0 +1,60 @@
'''
Onionr - Private P2P Communication
Remove a transport address but don't ban them
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
from onionrplugins import onionrevents as events
from onionrutils import stringvalidators
from onionrutils import mnemonickeys
from .. import dbfiles
from etc import onionrvalues
def remove_address(address):
'''
Remove an address from the address database
'''
if stringvalidators.validate_transport(address):
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
t = (address,)
c.execute('Delete from adders where address=?;', t)
conn.commit()
conn.close()
#events.event('address_remove', data = {'address': address}, onionr = core_inst.onionrInst)
return True
else:
return False
def remove_user(pubkey: str)->bool:
'''
Remove a user from the user database
'''
pubkey = mnemonickeys.get_base32(pubkey)
if stringvalidators.validate_pub_key(pubkey):
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
t = (pubkey,)
c.execute('Delete from peers where id=?;', t)
conn.commit()
conn.close()
return True
else:
return False

View file

@ -0,0 +1,75 @@
'''
Onionr - Private P2P Communication
get or set transport address meta information
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
from .. import dbfiles
from etc import onionrvalues
def get_address_info(address, info):
'''
Get info about an address from its database entry
address text, 0
type int, 1
knownPeer text, 2
speed int, 3
success int, 4
powValue 5
failure int 6
lastConnect 7
trust 8
introduced 9
'''
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
command = (address,)
infoNumbers = {'address': 0, 'type': 1, 'knownPeer': 2, 'speed': 3, 'success': 4, 'powValue': 5, 'failure': 6, 'lastConnect': 7, 'trust': 8, 'introduced': 9}
info = infoNumbers[info]
iterCount = 0
retVal = ''
for row in c.execute('SELECT * FROM adders WHERE address=?;', command):
for i in row:
if iterCount == info:
retVal = i
break
else:
iterCount += 1
conn.close()
return retVal
def set_address_info(address, key, data):
'''
Update an address for a key
'''
conn = sqlite3.connect(dbfiles.address_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
command = (data, address)
if key not in ('address', 'type', 'knownPeer', 'speed', 'success', 'failure', 'powValue', 'lastConnect', 'lastConnectAttempt', 'trust', 'introduced'):
raise ValueError("Got invalid database key when setting address info, must be in whitelist")
else:
c.execute('UPDATE adders SET ' + key + ' = ? WHERE address=?', command)
conn.commit()
conn.close()

View file

@ -0,0 +1,73 @@
'''
Onionr - Private P2P Communication
get or set information about a user id
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import sqlite3
from .. import dbfiles
from etc import onionrvalues
def get_user_info(peer, info):
'''
Get info about a peer from their database entry
id text 0
name text, 1
adders text, 2
dateSeen not null, 3
trust int 4
hashID text 5
'''
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
command = (peer,)
infoNumbers = {'id': 0, 'name': 1, 'adders': 2, 'dateSeen': 3, 'trust': 4, 'hashID': 5}
info = infoNumbers[info]
iterCount = 0
retVal = ''
for row in c.execute('SELECT * FROM peers WHERE id=?;', command):
for i in row:
if iterCount == info:
retVal = i
break
else:
iterCount += 1
conn.close()
return retVal
def set_peer_info(peer, key, data):
'''
Update a peer for a key
'''
conn = sqlite3.connect(dbfiles.user_id_info_db, timeout=onionrvalues.DATABASE_LOCK_TIMEOUT)
c = conn.cursor()
command = (data, peer)
if key not in ('id', 'name', 'pubkey', 'forwardKey', 'dateSeen', 'trust'):
raise ValueError("Got invalid database key when setting peer info")
c.execute('UPDATE peers SET ' + key + ' = ? WHERE id=?', command)
conn.commit()
conn.close()
set_user_info = set_peer_info