made storagecounter use a watchdog (inotify) instead of excessive file reads

master
Kevin 2020-07-24 02:57:44 -05:00
parent 8b5c87d356
commit f4d1739e4a
2 changed files with 62 additions and 32 deletions

View File

@ -1,8 +1,17 @@
""" """
Onionr - Private P2P Communication Onionr - Private P2P Communication.
Keeps track of how much disk space we're using Keep track of how much disk space we're using
""" """
from pathlib import Path
from threading import Thread
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
import config
from filepaths import usage_file
""" """
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@ -17,17 +26,46 @@
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
""" """
import config, filepaths
config.reload() config.reload()
def _read_data_file(f) -> int:
amount = 0
try:
with open(f, 'r') as f:
amount = int(f.read())
except FileNotFoundError:
pass
except ValueError:
pass # Possibly happens when the file is empty
return amount
class StorageCounter: class StorageCounter:
def __init__(self): def __init__(self):
self.data_file = filepaths.usage_file self.data_file = usage_file
return self.amount: int = None
Path(self.data_file).touch()
def is_full(self)->bool: def auto_refresher():
"""Returns if the allocated disk space is full (this is Onionr config, not true FS capacity)""" class Refresher(FileSystemEventHandler):
@staticmethod
def on_modified(event):
self.amount = _read_data_file(self.data_file)
observer = Observer()
observer.schedule(Refresher(), usage_file)
observer.start()
while observer.is_alive():
# call import func with timeout
observer.join(120)
Thread(target=auto_refresher, daemon=True).start()
self.amount = _read_data_file(self.data_file)
def is_full(self) -> bool:
"""Returns if the allocated disk space is full (this is Onionr config,
not true FS capacity)"""
ret_data = False ret_data = False
if config.get('allocations.disk', 1073741824) <= (self.get_amount() + 1000): if config.get('allocations.disk', 1073741824) <= (self.amount + 1000):
ret_data = True ret_data = True
return ret_data return ret_data
@ -35,26 +73,15 @@ class StorageCounter:
with open(self.data_file, 'w') as data_file: with open(self.data_file, 'w') as data_file:
data_file.write(str(data)) data_file.write(str(data))
def get_amount(self)->int: def get_percent(self) -> int:
"""Return how much disk space we're using (according to record)"""
ret_data = 0
try:
with open(self.data_file, 'r') as data_file:
ret_data = int(data_file.read())
except FileNotFoundError:
pass
except ValueError:
pass # Possibly happens when the file is empty
return ret_data
def get_percent(self)->int:
"""Return percent (decimal/float) of disk space we're using""" """Return percent (decimal/float) of disk space we're using"""
amount = self.get_amount() amount = self.amount
return round(amount / config.get('allocations.disk', 2000000000), 2) return round(amount / config.get('allocations.disk', 2000000000), 2)
def add_bytes(self, amount)->int: def add_bytes(self, amount) -> int:
"""Record that we are now using more disk space, unless doing so would exceed configured max""" """Record that we are now using more disk space,
new_amount = amount + self.get_amount() unless doing so would exceed configured max"""
new_amount = amount + self.amount
ret_data = new_amount ret_data = new_amount
if new_amount > config.get('allocations.disk', 2000000000): if new_amount > config.get('allocations.disk', 2000000000):
ret_data = False ret_data = False
@ -62,8 +89,8 @@ class StorageCounter:
self._update(new_amount) self._update(new_amount)
return ret_data return ret_data
def remove_bytes(self, amount)->int: def remove_bytes(self, amount) -> int:
"""Record that we are now using less disk space""" """Record that we are now using less disk space"""
new_amount = self.get_amount() - amount new_amount = self.amount - amount
self._update(new_amount) self._update(new_amount)
return new_amount return new_amount

View File

@ -2,6 +2,7 @@ import sys, os
sys.path.append(".") sys.path.append(".")
sys.path.append("src/") sys.path.append("src/")
import unittest, uuid import unittest, uuid
from time import sleep
TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/'
os.environ["ONIONR_HOME"] = TEST_DIR os.environ["ONIONR_HOME"] = TEST_DIR
@ -29,7 +30,7 @@ class TestStorageCounter(unittest.TestCase):
self.assertIsNotNone(config.get('allocations.disk')) self.assertIsNotNone(config.get('allocations.disk'))
self.assertGreaterEqual(config.get('allocations.disk'), 1000000) self.assertGreaterEqual(config.get('allocations.disk'), 1000000)
def test_insert_too_much(self): def test_insert_too_much(self):
_test_setup() _test_setup()
config.set('allocations.disk', 1000) config.set('allocations.disk', 1000)
@ -38,11 +39,13 @@ class TestStorageCounter(unittest.TestCase):
def test_count(self): def test_count(self):
_test_setup() _test_setup()
counter = storagecounter.StorageCounter() counter = storagecounter.StorageCounter()
start_value = counter.get_amount() start_value = counter.amount
b_hash = onionrblocks.insert("test") b_hash = onionrblocks.insert("test")
self.assertGreater(counter.get_amount(), start_value) sleep(0.1)
self.assertGreater(counter.amount, start_value)
onionrstorage.removeblock.remove_block(b_hash) onionrstorage.removeblock.remove_block(b_hash)
self.assertEqual(counter.get_amount(), start_value) sleep(0.1)
self.assertEqual(counter.amount, start_value)
unittest.main() unittest.main()