From c23b286540b6fcc8508cdcf60bffcdf45517f70a Mon Sep 17 00:00:00 2001 From: Kevin Froman Date: Sat, 17 Oct 2020 07:51:28 +0000 Subject: [PATCH] added file reader offset --- .../private/register_private_blueprints.py | 2 ++ src/httpapi/fileoffsetreader/__init__.py | 30 +++++++++++++++++++ src/utils/readoffset.py | 17 +++++++++++ src/utils/readstatic.py | 4 +-- tests/test_read_offset.py | 29 ++++++++++++++++++ 5 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 src/httpapi/fileoffsetreader/__init__.py create mode 100644 src/utils/readoffset.py create mode 100644 tests/test_read_offset.py diff --git a/src/apiservers/private/register_private_blueprints.py b/src/apiservers/private/register_private_blueprints.py index 8b6857b6..28d222ff 100644 --- a/src/apiservers/private/register_private_blueprints.py +++ b/src/apiservers/private/register_private_blueprints.py @@ -9,6 +9,7 @@ from httpapi import security, friendsapi, profilesapi, configapi, insertblock from httpapi import miscclientapi, onionrsitesapi, apiutils from httpapi import directconnections from httpapi import themeapi +from httpapi import fileoffsetreader from httpapi.sse.private import private_sse_blueprint """ @@ -46,6 +47,7 @@ def register_private_blueprints(private_api, app): private_api).direct_conn_management_bp) app.register_blueprint(themeapi.theme_blueprint) app.register_blueprint(private_sse_blueprint) + app.register_blueprint(fileoffsetreader.offset_reader_api) def _add_events_bp(): while True: diff --git a/src/httpapi/fileoffsetreader/__init__.py b/src/httpapi/fileoffsetreader/__init__.py new file mode 100644 index 00000000..ab427d36 --- /dev/null +++ b/src/httpapi/fileoffsetreader/__init__.py @@ -0,0 +1,30 @@ +from os.path import exists, dirname + +import ujson +from flask import Blueprint, Response, request + +from utils.identifyhome import identify_home +from utils.readoffset import read_from_offset + +offset_reader_api = Blueprint('offsetreaderapi', __name__) + + +@offset_reader_api.route('/readfileoffset/') +def offset_reader_endpoint(name): + if not name[:-4].isalnum(): + return Response(400, "Path must be alphanumeric except for file ext") + + path = identify_home() + name + + if not exists(path): + return Response(404, "Path not found in Onionr data directory") + + offset = request.args.get('offset') + + if not offset: + offset = 0 + else: + offset = int(offset) + result = read_from_offset(path, offset)._asdict() + + return ujson.dumps(result, reject_bytes=False) diff --git a/src/utils/readoffset.py b/src/utils/readoffset.py new file mode 100644 index 00000000..ecfcba4f --- /dev/null +++ b/src/utils/readoffset.py @@ -0,0 +1,17 @@ +"""Onionr - Private P2P Communication. + +read from a file from an offset (efficiently) +""" +from collections import namedtuple + +OffsetReadResult = namedtuple('OffsetReadResult', ['data', 'new_offset']) + + +def read_from_offset(file_path, offset=0): + with open(file_path, 'rb') as f: + if offset: + f.seek(offset) + data = f.read() + offset = f.tell() + + return OffsetReadResult(data, offset) diff --git a/src/utils/readstatic.py b/src/utils/readstatic.py index 59b331c0..4d0f579c 100644 --- a/src/utils/readstatic.py +++ b/src/utils/readstatic.py @@ -20,11 +20,11 @@ import os """ -def get_static_dir()->str: +def get_static_dir() -> str: return os.path.dirname(os.path.realpath(__file__)) + '/../../static-data/' -def read_static(file:str, ret_bin:bool=False)->Union[str, bytes]: +def read_static(file: str, ret_bin: bool = False) -> Union[str, bytes]: static_file = get_static_dir() + file if ret_bin: diff --git a/tests/test_read_offset.py b/tests/test_read_offset.py new file mode 100644 index 00000000..b7832667 --- /dev/null +++ b/tests/test_read_offset.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +import sys, os +from time import sleep +import tempfile +sys.path.append(".") +sys.path.append("src/") +import uuid +TEST_DIR = 'testdata/%s-%s' % (uuid.uuid4(), os.path.basename(__file__)) + '/' +print("Test directory:", TEST_DIR) +os.environ["ONIONR_HOME"] = TEST_DIR +import unittest, json + +from utils import identifyhome, createdirs, readoffset +from onionrsetup import setup_config +createdirs.create_dirs() +setup_config() + +class TestReadOffset(unittest.TestCase): + def test_read_whole(self): + temp = tempfile.mkstemp()[1] + f = open(temp, 'wb') + data = b"test1\ntest2\ntest3\test4" + f.write(data) + f.close() + self.assertEqual(readoffset.read_from_offset(temp).data, data) + os.remove(temp) + + +unittest.main()