From 9587085e8e6f866f3aa12bb640007197a0bb5ba4 Mon Sep 17 00:00:00 2001 From: Tom Marks Date: Thu, 24 May 2018 17:35:00 -0400 Subject: [PATCH 1/4] Updates .gitignore --- .gitignore | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index b860bf3..9e0cd82 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,104 @@ -*.egg-info -/dist/ -/build/ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ .coverage .coverage.* -.tox +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# visual studio code +.vscode \ No newline at end of file From 36843c45455cebf05c7cf4f889392a31e88e9bd5 Mon Sep 17 00:00:00 2001 From: Tom Marks Date: Thu, 24 May 2018 17:35:29 -0400 Subject: [PATCH 2/4] Updates pack_uwsgi_vars Old packing of uwsgi vars seemed a little bit like magic and was hard to understand at least for mere mortals like myself. Created a pair of C Structures using definitions on uwsgi site and a helper function to convert to bytes. --- tests/test_uwsgi_structs.py | 55 ++++++++++++++++++++++++++++++++++ uwsgi_tools/compat.py | 8 ++--- uwsgi_tools/utils.py | 23 ++++++++------- uwsgi_tools/uwsgi_structs.py | 57 ++++++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 15 deletions(-) create mode 100644 tests/test_uwsgi_structs.py create mode 100644 uwsgi_tools/uwsgi_structs.py diff --git a/tests/test_uwsgi_structs.py b/tests/test_uwsgi_structs.py new file mode 100644 index 0000000..8fa6618 --- /dev/null +++ b/tests/test_uwsgi_structs.py @@ -0,0 +1,55 @@ +import unittest + +from uwsgi_tools import uwsgi_structs as uw + +""" +From definitions available at +http://uwsgi-docs.readthedocs.io/en/latest/Protocol.html +""" + + +class TestUwsgiPacketHeader(unittest.TestCase): + header = uw.UwsgiPacketHeader(0, 166, 0) + header_bytes = b'\x00\xa6\x00\x00' + + def test_packet_header_serializes(self): + expected = self.header_bytes + + self.assertEqual(expected, bytearray(self.header)) + + def test_packet_header_deserializes(self): + buffer = bytearray(self.header_bytes) + expected_header = uw.UwsgiPacketHeader(0, 166, 0) + + header = uw.UwsgiPacketHeader.from_buffer(buffer) + self.assertEqual( + bytearray(expected_header), + bytearray(header) + ) + + +class TestUwsgiVar(unittest.TestCase): + + key = b'foo' + key_size = len(key) + val = b'bar' + val_size = len(val) + + var = uw.UwsgiVar(key_size, key, val_size, val) + var_bytes = b'\x03\x00foo\x03\x00bar' + + def test_var_serializes(self): + self.assertEqual( + self.var_bytes, + bytearray(self.var) + ) + + def test_var_deserializes(self): + buffer = bytearray(self.var_bytes) + expected = self.var + + var = uw.UwsgiVar.from_buffer(buffer) + self.assertEqual( + bytearray(expected), + bytearray(var) + ) diff --git a/uwsgi_tools/compat.py b/uwsgi_tools/compat.py index 521b979..aa48279 100644 --- a/uwsgi_tools/compat.py +++ b/uwsgi_tools/compat.py @@ -2,7 +2,7 @@ __all__ = [ 'BaseHTTPRequestHandler', 'TCPServer', 'get_content_type', 'urlsplit', - 'hex2bytes', + 'struct2bytes', ] PY3 = sys.version_info[0] == 3 @@ -25,8 +25,8 @@ def get_content_type(headers): return headers.typeheader -def hex2bytes(s): +def struct2bytes(s): if PY3: - return bytes.fromhex(s) + return bytes(s) else: - return s.decode('hex') + return bytes(bytearray(s)) diff --git a/uwsgi_tools/utils.py b/uwsgi_tools/utils.py index 1b153c4..9b324a9 100644 --- a/uwsgi_tools/utils.py +++ b/uwsgi_tools/utils.py @@ -1,17 +1,18 @@ -from .compat import hex2bytes, urlsplit - - -def sz(x): - s = hex(x if isinstance(x, int) else len(x))[2:].rjust(4, '0') - s = hex2bytes(s) - return s[::-1] +from .compat import struct2bytes, urlsplit +from .uwsgi_structs import UwsgiPacketHeader, UwsgiVar def pack_uwsgi_vars(var): - pk = b'' - for k, v in var.items() if hasattr(var, 'items') else var: - pk += sz(k) + k.encode('utf8') + sz(v) + v.encode('utf8') - return b'\x00' + sz(pk) + b'\x00' + pk + encoded_vars = [ + (k.encode('utf-8'), v.encode('utf-8')) + for k, v in var.items() + ] + packed_vars = b''.join( + struct2bytes(UwsgiVar(len(k), k, len(v), v)) + for k, v in encoded_vars + ) + packet_header = struct2bytes(UwsgiPacketHeader(0, len(packed_vars), 0)) + return packet_header + packed_vars def parse_addr(addr, default_port=3030): diff --git a/uwsgi_tools/uwsgi_structs.py b/uwsgi_tools/uwsgi_structs.py new file mode 100644 index 0000000..73d0982 --- /dev/null +++ b/uwsgi_tools/uwsgi_structs.py @@ -0,0 +1,57 @@ +""" +From definitions available at +http://uwsgi-docs.readthedocs.io/en/latest/Protocol.html +""" + +import ctypes + + +class UwsgiPacketHeader(ctypes.Structure): + """ + struct uwsgi_packet_header { + uint8_t modifier1; + uint16_t datasize; + uint8_t modifier2; + }; + """ + _pack_ = 1 + _fields_ = [ + ("modifier1", ctypes.c_int8), + ("datasize", ctypes.c_int16), + ("modifier2", ctypes.c_int8), + ] + + +class UwsgiVar(object): + """ + struct uwsgi_var { + uint16_t key_size; + uint8_t key[key_size]; + uint16_t val_size; + uint8_t val[val_size]; + } + """ + + def __new__(self, key_size, key, val_size, val): + class UwsgiVar(ctypes.Structure): + _pack_ = 1 + _fields_ = [ + ("key_size", ctypes.c_int16), + ("key", ctypes.c_char * key_size), + ("val_size", ctypes.c_int16), + ("val", ctypes.c_char * val_size), + ] + + return UwsgiVar(key_size, key, val_size, val) + + @classmethod + def from_buffer(cls, buffer, offset=0): + key_size = ctypes.c_int16.from_buffer(buffer, offset).value + offset += ctypes.sizeof(ctypes.c_int16) + key = (ctypes.c_char * key_size).from_buffer(buffer, offset).value + offset += ctypes.sizeof(ctypes.c_char * key_size) + val_size = ctypes.c_int16.from_buffer(buffer, offset).value + offset += ctypes.sizeof(ctypes.c_int16) + val = (ctypes.c_char * val_size).from_buffer(buffer, offset).value + + return cls(key_size, key, val_size, val) From cc4015fe472e1510c148b701d952e0ab2e533ea7 Mon Sep 17 00:00:00 2001 From: Tom Marks Date: Thu, 24 May 2018 18:30:32 -0400 Subject: [PATCH 3/4] Adds CONTENT_LENGTH & CONTENT_TYPE so that post requests work correctly --- tests/test_curl.py | 21 +++++++++++++++++++++ uwsgi_tools/curl.py | 21 ++++++++++++++++----- uwsgi_tools/proxy.py | 2 +- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/tests/test_curl.py b/tests/test_curl.py index caf8d3d..dcdd437 100644 --- a/tests/test_curl.py +++ b/tests/test_curl.py @@ -1,3 +1,4 @@ +from copy import deepcopy import socket import unittest from uwsgi_tools.curl import cli @@ -22,3 +23,23 @@ def test_file_socket(self): def test_headers(self): with server(callback=lambda x: self.assertIn(b'localhost', x)): self.assertFalse(cli('127.0.0.1:3030', '-H', 'Host: localhost')) + + def test_post(self): + requests = [] + + def request_sniper(x): + requests.append(deepcopy(x)) + + with server(callback=request_sniper): + self.assertFalse(cli( + '--method', 'POST', + '--header', 'Content-Type: application/json', + r'''--data=\'{"foo":"bar"}\'''', + '127.0.0.1', + 'host.name/')) + + self.assertIn(b'POST', requests[0]) + # Magic number is the val_size + val + self.assertIn(b'CONTENT_LENGTH\x02\x0017', requests[0]) + self.assertIn(b'CONTENT_TYPE', requests[0]) + self.assertIn(b'application/json', requests[0]) diff --git a/uwsgi_tools/curl.py b/uwsgi_tools/curl.py index 1cd7b95..eda10f4 100644 --- a/uwsgi_tools/curl.py +++ b/uwsgi_tools/curl.py @@ -17,11 +17,8 @@ def ask_uwsgi(uwsgi_addr, var, body='', timeout=0, udp=False): if timeout: s.settimeout(timeout) - if body is None: - body = '' - s.connect(addr) - s.send(pack_uwsgi_vars(var) + body.encode('utf8')) + s.send(pack_uwsgi_vars(var) + body) response = [] while 1: data = s.recv(4096) @@ -45,6 +42,8 @@ def curl(uwsgi_addr, url, method='GET', body='', timeout=0, headers=(), else: port = None + body = (body or '').encode('utf-8') + var = { 'SERVER_PROTOCOL': 'HTTP/1.1', 'PATH_INFO': parts_uri.path, @@ -52,10 +51,22 @@ def curl(uwsgi_addr, url, method='GET', body='', timeout=0, headers=(), 'REQUEST_URI': uri, 'QUERY_STRING': parts_uri.query, 'HTTP_HOST': host, + 'CONTENT_LENGTH': str(len(body)), + # Other varaibles seen in nginx's uwsgi_params file but not explicitly + # handled anywhere in this file + # https://github.com/nginx/nginx/blob/master/conf/uwsgi_params + # DOCUMENT_ROOT + # REQUEST_SCHEME + # HTTPS + # REMOTE_ADDR + # REMOTE_PORT } + for header in headers or (): key, _, value = header.partition(':') - var['HTTP_' + key.strip().upper()] = value.strip() + var['HTTP_' + key.strip().upper().replace('-', '_')] = value.strip() + if 'HTTP_CONTENT_TYPE' in var.keys(): + var['CONTENT_TYPE'] = var['HTTP_CONTENT_TYPE'] var['SERVER_NAME'] = var['HTTP_HOST'] if port: var['SERVER_PORT'] = str(port) diff --git a/uwsgi_tools/proxy.py b/uwsgi_tools/proxy.py index 182fe75..9b05fed 100644 --- a/uwsgi_tools/proxy.py +++ b/uwsgi_tools/proxy.py @@ -41,7 +41,7 @@ def do(self): env.pop('HTTP_REFERER', 0) cl = env['CONTENT_LENGTH'] - body = repr(self.rfile.read(int(cl))) if cl else '' + body = (repr(self.rfile.read(int(cl))) if cl else '').encode('utf-8') resp = ask_uwsgi((self.server.uwsgi_addr, self.server.uwsgi_port), var=env, body=body) From 20dec3008dfd2162a9307560fb2c9c3407b86251 Mon Sep 17 00:00:00 2001 From: Tom Marks Date: Fri, 25 May 2018 11:33:14 -0400 Subject: [PATCH 4/4] Updated cli parser to load arguments which start with @ from file --- tests/test_curl.py | 29 +++++++++++++++++++++++++++++ uwsgi_tools/curl.py | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/test_curl.py b/tests/test_curl.py index dcdd437..608a2e2 100644 --- a/tests/test_curl.py +++ b/tests/test_curl.py @@ -1,11 +1,19 @@ from copy import deepcopy +import json import socket +import tempfile import unittest + from uwsgi_tools.curl import cli from tests.utils import server class CurlTests(unittest.TestCase): + def setUp(self): + self.datafile = tempfile.NamedTemporaryFile(delete=True) + with open(self.datafile.name, 'w') as fh: + json.dump({'foo': 'bar'}, fh) + def test_cli(self): with server(): self.assertFalse(cli('127.0.0.1', 'host.name/')) @@ -43,3 +51,24 @@ def request_sniper(x): self.assertIn(b'CONTENT_LENGTH\x02\x0017', requests[0]) self.assertIn(b'CONTENT_TYPE', requests[0]) self.assertIn(b'application/json', requests[0]) + + def test_at_prefixed_data(self): + requests = [] + + def request_sniper(x): + requests.append(deepcopy(x)) + + with server(callback=request_sniper): + self.assertFalse(cli( + '--method', 'POST', + '--header', 'Content-Type: application/json', + '--data', '@{}'.format(self.datafile.name), + '127.0.0.1', + 'host.name/', + )) + + self.assertIn(b'POST', requests[0]) + # Magic number is the val_size + val + self.assertIn(b'CONTENT_LENGTH\x02\x0014', requests[0]) + self.assertIn(b'CONTENT_TYPE', requests[0]) + self.assertIn(b'application/json', requests[0]) diff --git a/uwsgi_tools/curl.py b/uwsgi_tools/curl.py index eda10f4..d5f790c 100644 --- a/uwsgi_tools/curl.py +++ b/uwsgi_tools/curl.py @@ -78,7 +78,7 @@ def curl(uwsgi_addr, url, method='GET', body='', timeout=0, headers=(), def cli(*args): import argparse - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(fromfile_prefix_chars='@') parser.add_argument('uwsgi_addr', nargs=1, help="Remote address of uWSGI server")