From b14db9a0ae09c2eca2d6d5862d6dedda576e86af Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Tue, 29 Oct 2024 22:43:54 -0400 Subject: [PATCH 01/15] =?UTF-8?q?=F0=9F=8E=A8=20run=20black?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiderweb/middleware/gzip.py | 38 ++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/spiderweb/middleware/gzip.py b/spiderweb/middleware/gzip.py index cdf55a1..1f956a1 100644 --- a/spiderweb/middleware/gzip.py +++ b/spiderweb/middleware/gzip.py @@ -1,36 +1,36 @@ - """ - Source code inspiration :https://github.com/colour-science/flask-compress/blob/master/flask_compress/flask_compress.py +Source code inspiration: https://github.com/colour-science/flask-compress/blob/master/flask_compress/flask_compress.py """ - from spiderweb.middleware import SpiderwebMiddleware from spiderweb.request import Request from spiderweb.response import HttpResponse - import gzip class GzipMiddleware(SpiderwebMiddleware): - + algorithm = "gzip" minimum_length = 500 - def post_process(self, request: Request, response: HttpResponse, rendered_response: str) -> str: - - #right status, length > 500, instance string (because FileResponse returns list of bytes , + def post_process( + self, request: Request, response: HttpResponse, rendered_response: str + ) -> str: + + # right status, length > 500, instance string (because FileResponse returns list of bytes, # not already compressed, and client accepts gzip - if not (200 <= response.status_code < 300) or \ - len(rendered_response) < self.minimum_length or \ - not isinstance(rendered_response, str) or \ - self.algorithm in response.headers.get("Content-Encoding", "") or \ - self.algorithm not in request.headers.get("Accept-Encoding", ""): - return rendered_response - - zipped = gzip.compress(rendered_response.encode('UTF-8')) + if ( + not (200 <= response.status_code < 300) + or len(rendered_response) < self.minimum_length + or not isinstance(rendered_response, str) + or self.algorithm in response.headers.get("Content-Encoding", "") + or self.algorithm not in request.headers.get("Accept-Encoding", "") + ): + return rendered_response + + zipped = gzip.compress(rendered_response.encode("UTF-8"), compresslevel=6) response.headers["Content-Encoding"] = self.algorithm response.headers["Content-Length"] = str(len(zipped)) - - return zipped - + + return zipped.decode("UTF-8") From 236fc84be1fa5fae35a200a52e895599267e0077 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Tue, 29 Oct 2024 22:49:05 -0400 Subject: [PATCH 02/15] =?UTF-8?q?=F0=9F=93=9D=20clarify=20inline=20comment?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiderweb/middleware/gzip.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/spiderweb/middleware/gzip.py b/spiderweb/middleware/gzip.py index 1f956a1..6589c7c 100644 --- a/spiderweb/middleware/gzip.py +++ b/spiderweb/middleware/gzip.py @@ -17,9 +17,13 @@ class GzipMiddleware(SpiderwebMiddleware): def post_process( self, request: Request, response: HttpResponse, rendered_response: str ) -> str: - - # right status, length > 500, instance string (because FileResponse returns list of bytes, - # not already compressed, and client accepts gzip + # Only actually compress the response if the following attributes are true: + # + # * The response status code is a 2xx success code + # * The response length is at least 500 bytes + # * The response is not already compressed (e.g. it's not an image) + # * The request accepts gzip encoding + # * The response is not a streaming response if ( not (200 <= response.status_code < 300) or len(rendered_response) < self.minimum_length From 203b4f7e0fdb189d508fe4f42380dcea4e454a3e Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Tue, 29 Oct 2024 23:27:43 -0400 Subject: [PATCH 03/15] =?UTF-8?q?=E2=9C=A8=20add=20gzip=20middleware=20to?= =?UTF-8?q?=20example=20server?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- example.py | 1 + 1 file changed, 1 insertion(+) diff --git a/example.py b/example.py index 72a0cd2..c1745e2 100644 --- a/example.py +++ b/example.py @@ -15,6 +15,7 @@ from spiderweb.response import ( app = SpiderwebRouter( templates_dirs=["templates"], middleware=[ + "spiderweb.middleware.gzip.GzipMiddleware", "spiderweb.middleware.cors.CorsMiddleware", "spiderweb.middleware.sessions.SessionMiddleware", "spiderweb.middleware.csrf.CSRFMiddleware", From 6c2cfc5297d3c8abdaa795dfe52316600aedf3d5 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Tue, 29 Oct 2024 23:28:40 -0400 Subject: [PATCH 04/15] =?UTF-8?q?=E2=9C=85=20add=20new=20user=20settings?= =?UTF-8?q?=20and=20server=20checks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiderweb/main.py | 5 +++++ spiderweb/middleware/gzip.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/spiderweb/main.py b/spiderweb/main.py index 82ab30a..51230a4 100644 --- a/spiderweb/main.py +++ b/spiderweb/main.py @@ -69,6 +69,8 @@ class SpiderwebRouter(LocalServerMixin, MiddlewareMixin, RoutesMixin, FernetMixi csrf_trusted_origins: Sequence[str] = None, db: Optional[Database] = None, debug: bool = False, + gzip_compression_level: int = 6, + gzip_minimum_response_length: int = 500, templates_dirs: Sequence[str] = None, middleware: Sequence[str] = None, append_slash: bool = False, @@ -119,6 +121,9 @@ class SpiderwebRouter(LocalServerMixin, MiddlewareMixin, RoutesMixin, FernetMixi convert_url_to_regex(i) for i in self._csrf_trusted_origins ] + self.gzip_compression_level = gzip_compression_level + self.gzip_minimum_response_length = gzip_minimum_response_length + self.debug = debug self.extra_data = kwargs diff --git a/spiderweb/middleware/gzip.py b/spiderweb/middleware/gzip.py index 6589c7c..00982b1 100644 --- a/spiderweb/middleware/gzip.py +++ b/spiderweb/middleware/gzip.py @@ -1,16 +1,41 @@ """ Source code inspiration: https://github.com/colour-science/flask-compress/blob/master/flask_compress/flask_compress.py """ - +from spiderweb.exceptions import ConfigError from spiderweb.middleware import SpiderwebMiddleware +from spiderweb.server_checks import ServerCheck from spiderweb.request import Request from spiderweb.response import HttpResponse import gzip +class CheckValidGzipCompressionLevel(ServerCheck): + INVALID_GZIP_COMPRESSION_LEVEL = ( + "`gzip_compression_level` must be an integer between 1 and 9." + ) + + def check(self): + if not isinstance(self.server.gzip_compression_level, int): + raise ConfigError(self.INVALID_GZIP_COMPRESSION_LEVEL) + if self.server.gzip_compression_level not in range(1, 10): + raise ConfigError("Gzip compression level must be an integer between 1 and 9.") + + +class CheckValidGzipMinimumLength(ServerCheck): + INVALID_GZIP_MINIMUM_LENGTH = "`gzip_minimum_length` must be a positive integer." + + def check(self): + if not isinstance(self.server.gzip_minimum_length, int): + raise ConfigError(self.INVALID_GZIP_MINIMUM_LENGTH) + if self.server.gzip_minimum_length < 1: + raise ConfigError(self.INVALID_GZIP_MINIMUM_LENGTH) + + class GzipMiddleware(SpiderwebMiddleware): + checks = [CheckValidGzipCompressionLevel, CheckValidGzipMinimumLength] + algorithm = "gzip" minimum_length = 500 @@ -37,4 +62,4 @@ class GzipMiddleware(SpiderwebMiddleware): response.headers["Content-Encoding"] = self.algorithm response.headers["Content-Length"] = str(len(zipped)) - return zipped.decode("UTF-8") + return zipped From 98ca09b6813d16d382bb55b1e5ac858d574f2f75 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Tue, 29 Oct 2024 23:30:26 -0400 Subject: [PATCH 05/15] =?UTF-8?q?=F0=9F=8F=B7=EF=B8=8F=20add=20type=20hint?= =?UTF-8?q?s=20and=20docstring=20updates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiderweb/middleware/base.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/spiderweb/middleware/base.py b/spiderweb/middleware/base.py index e9a22ff..4c1dabf 100644 --- a/spiderweb/middleware/base.py +++ b/spiderweb/middleware/base.py @@ -1,6 +1,11 @@ +from typing import TYPE_CHECKING + from spiderweb.request import Request from spiderweb.response import HttpResponse +if TYPE_CHECKING: + from spiderweb.server_checks import ServerCheck + class SpiderwebMiddleware: """ @@ -22,6 +27,10 @@ class SpiderwebMiddleware: def __init__(self, server): self.server = server + # If there are any startup checks that need to be run, they should be added + # to this list. These checks should be classes that inherit from + # spiderweb.server_checks.ServerCheck. + self.checks: list[ServerCheck] = [] def process_request(self, request: Request) -> HttpResponse | None: # This method is called before the request is passed to the view. You can safely @@ -45,5 +54,6 @@ class SpiderwebMiddleware: self, request: Request, response: HttpResponse, rendered_response: str ) -> str: # This method is called after all the middleware has been processed and receives - # the final rendered response in str form. You can modify the response here. + # the final rendered response in str form. You can modify the response here. This + # method *must* return a str version of the rendered response. return rendered_response From dca1b89b39167e6958316a6872d34d1310406747 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Tue, 29 Oct 2024 23:32:02 -0400 Subject: [PATCH 06/15] =?UTF-8?q?=F0=9F=90=9B=20don't=20actually=20clear?= =?UTF-8?q?=20out=20the=20check=20list=20lol?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiderweb/middleware/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spiderweb/middleware/base.py b/spiderweb/middleware/base.py index 4c1dabf..82db29e 100644 --- a/spiderweb/middleware/base.py +++ b/spiderweb/middleware/base.py @@ -30,7 +30,7 @@ class SpiderwebMiddleware: # If there are any startup checks that need to be run, they should be added # to this list. These checks should be classes that inherit from # spiderweb.server_checks.ServerCheck. - self.checks: list[ServerCheck] = [] + self.checks: list[ServerCheck] def process_request(self, request: Request) -> HttpResponse | None: # This method is called before the request is passed to the view. You can safely From 707a3a82c3956b2cf935e88c5b6c7993a730c1b9 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Tue, 29 Oct 2024 23:50:07 -0400 Subject: [PATCH 07/15] =?UTF-8?q?=F0=9F=8F=B7=EF=B8=8F=20fix=20types=20to?= =?UTF-8?q?=20be=20explicit=20about=20bytes=20being=20okay?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiderweb/middleware/__init__.py | 2 +- spiderweb/middleware/gzip.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spiderweb/middleware/__init__.py b/spiderweb/middleware/__init__.py index 9118034..9ebff28 100644 --- a/spiderweb/middleware/__init__.py +++ b/spiderweb/middleware/__init__.py @@ -63,7 +63,7 @@ class MiddlewareMixin: def post_process_middleware( self, request: Request, response: HttpResponse, rendered_response: str - ) -> str: + ) -> str | bytes: # run them in reverse order, same as process_response. The top of the middleware # stack should be the first and last middleware to run. for middleware in reversed(self.middleware): diff --git a/spiderweb/middleware/gzip.py b/spiderweb/middleware/gzip.py index 00982b1..6610a6d 100644 --- a/spiderweb/middleware/gzip.py +++ b/spiderweb/middleware/gzip.py @@ -41,7 +41,7 @@ class GzipMiddleware(SpiderwebMiddleware): def post_process( self, request: Request, response: HttpResponse, rendered_response: str - ) -> str: + ) -> str | bytes: # Only actually compress the response if the following attributes are true: # # * The response status code is a 2xx success code From 95f9479aa9365aa3d6edc313a8080bd0e986017e Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Tue, 29 Oct 2024 23:50:32 -0400 Subject: [PATCH 08/15] =?UTF-8?q?=F0=9F=8E=A8=20prettify=20a=20little?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiderweb/middleware/gzip.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/spiderweb/middleware/gzip.py b/spiderweb/middleware/gzip.py index 6610a6d..5283560 100644 --- a/spiderweb/middleware/gzip.py +++ b/spiderweb/middleware/gzip.py @@ -19,7 +19,9 @@ class CheckValidGzipCompressionLevel(ServerCheck): if not isinstance(self.server.gzip_compression_level, int): raise ConfigError(self.INVALID_GZIP_COMPRESSION_LEVEL) if self.server.gzip_compression_level not in range(1, 10): - raise ConfigError("Gzip compression level must be an integer between 1 and 9.") + raise ConfigError( + "Gzip compression level must be an integer between 1 and 9." + ) class CheckValidGzipMinimumLength(ServerCheck): @@ -44,11 +46,12 @@ class GzipMiddleware(SpiderwebMiddleware): ) -> str | bytes: # Only actually compress the response if the following attributes are true: # - # * The response status code is a 2xx success code - # * The response length is at least 500 bytes - # * The response is not already compressed (e.g. it's not an image) - # * The request accepts gzip encoding - # * The response is not a streaming response + # - The response status code is a 2xx success code + # - The response length is at least 500 bytes + # - The response is not a streaming response + # - (already bytes, like from FileResponse) + # - The response is not already compressed + # - The request accepts gzip encoding if ( not (200 <= response.status_code < 300) or len(rendered_response) < self.minimum_length From 972225d8bc773c77fa883720a0487fbe4e08d126 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Tue, 29 Oct 2024 23:50:53 -0400 Subject: [PATCH 09/15] =?UTF-8?q?=E2=9C=A8=20actually=20use=20those=20shin?= =?UTF-8?q?y=20new=20config=20settings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiderweb/middleware/gzip.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/spiderweb/middleware/gzip.py b/spiderweb/middleware/gzip.py index 5283560..68a9001 100644 --- a/spiderweb/middleware/gzip.py +++ b/spiderweb/middleware/gzip.py @@ -1,6 +1,7 @@ """ Source code inspiration: https://github.com/colour-science/flask-compress/blob/master/flask_compress/flask_compress.py """ + from spiderweb.exceptions import ConfigError from spiderweb.middleware import SpiderwebMiddleware from spiderweb.server_checks import ServerCheck @@ -28,9 +29,9 @@ class CheckValidGzipMinimumLength(ServerCheck): INVALID_GZIP_MINIMUM_LENGTH = "`gzip_minimum_length` must be a positive integer." def check(self): - if not isinstance(self.server.gzip_minimum_length, int): + if not isinstance(self.server.gzip_minimum_response_length, int): raise ConfigError(self.INVALID_GZIP_MINIMUM_LENGTH) - if self.server.gzip_minimum_length < 1: + if self.server.gzip_minimum_response_length < 1: raise ConfigError(self.INVALID_GZIP_MINIMUM_LENGTH) @@ -39,7 +40,6 @@ class GzipMiddleware(SpiderwebMiddleware): checks = [CheckValidGzipCompressionLevel, CheckValidGzipMinimumLength] algorithm = "gzip" - minimum_length = 500 def post_process( self, request: Request, response: HttpResponse, rendered_response: str @@ -54,15 +54,17 @@ class GzipMiddleware(SpiderwebMiddleware): # - The request accepts gzip encoding if ( not (200 <= response.status_code < 300) - or len(rendered_response) < self.minimum_length + or len(rendered_response) < self.server.gzip_minimum_response_length or not isinstance(rendered_response, str) or self.algorithm in response.headers.get("Content-Encoding", "") or self.algorithm not in request.headers.get("Accept-Encoding", "") ): return rendered_response - zipped = gzip.compress(rendered_response.encode("UTF-8"), compresslevel=6) + zipped = gzip.compress( + rendered_response.encode("UTF-8"), + compresslevel=self.server.gzip_compression_level, + ) response.headers["Content-Encoding"] = self.algorithm response.headers["Content-Length"] = str(len(zipped)) - return zipped From b9ad2467df30654646dae51c52a3aefe4c2e8dae Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Tue, 29 Oct 2024 23:51:13 -0400 Subject: [PATCH 10/15] =?UTF-8?q?=F0=9F=93=9D=20clarify=20custom=20post=5F?= =?UTF-8?q?process=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/middleware/custom_middleware.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/middleware/custom_middleware.md b/docs/middleware/custom_middleware.md index 7cfa8a2..a195a70 100644 --- a/docs/middleware/custom_middleware.md +++ b/docs/middleware/custom_middleware.md @@ -57,7 +57,7 @@ Unlike `process_request`, returning a value here doesn't change anything. We're This is a helper function that is available for you to override; it's not often used by middleware, but there are some ([like the pydantic middleware](middleware/pydantic.md)) that call `on_error` when there is a validation failure. -## post_process(self, request: Request, response: HttpResponse, rendered_response: str) -> str: +## post_process(self, request: Request, response: HttpResponse, rendered_response: str) -> str | bytes: > New in 1.3.0! @@ -67,7 +67,7 @@ There are three things passed to `post_process`: - `request`: the request object. It's provided here purely for reference purposes; while you can technically change it here, it won't have any effect on the response. - `response`: the response object. The full HTML of the response has already been rendered, but the headers can still be modified here. This object can be modified in place, like in `process_response`. -- `rendered_response`: the full HTML of the response as a string. This is the final output that will be sent to the client. Every instance of `post_process` must return the full HTML of the response, so if you want to make changes, you'll need to return the modified string. +- `rendered_response`: the full HTML of the response as a string or bytes. This is the final output that will be sent to the client. Every instance of `post_process` must return the full HTML of the response, so if you want to make changes, you'll need to return the modified string. A string is _strongly_ preferred, but bytes are also acceptable; keep in mind that you'll be making things harder for any `post_process` middleware that comes after you. Note that this function *must* return the full HTML of the response (provided at the start as `rendered_response`. Each invocation of `post_process` overwrites the entire output of the response, so make sure to return everything that you want to send. For example, here's a middleware that ~~breaks~~ adjusts the capitalization of the response and also demonstrates passing variables into the middleware and modifies the headers with the type of transformation: @@ -75,13 +75,13 @@ Note that this function *must* return the full HTML of the response (provided at import random from spiderweb.request import Request +from spiderweb.response import HttpResponse from spiderweb.middleware import SpiderwebMiddleware from spiderweb.exceptions import ConfigError class CaseTransformMiddleware(SpiderwebMiddleware): - # this breaks everything, but it's hilarious so it's worth it. - # Blame Sam. + # this breaks everything, but it's hilarious so it's worth it. Blame Sam. def post_process(self, request: Request, response: HttpResponse, rendered_response: str) -> str: valid_options = ["spongebob", "random"] # grab the value from the extra data passed into the server object @@ -109,6 +109,7 @@ class CaseTransformMiddleware(SpiderwebMiddleware): ) # usage: +from spiderweb import SpiderwebRouter app = SpiderwebRouter( middleware=["CaseTransformMiddleware"], From f4ffa14b0004cf9c1ec189d89d4c4297982f6a77 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Wed, 30 Oct 2024 00:15:32 -0400 Subject: [PATCH 11/15] =?UTF-8?q?=E2=9C=85=20add=20tests=20to=20cover=20ne?= =?UTF-8?q?w=20middleware?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiderweb/tests/test_middleware.py | 64 +++++++++++++++++++++++++++++- spiderweb/tests/views_for_tests.py | 7 +++- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/spiderweb/tests/test_middleware.py b/spiderweb/tests/test_middleware.py index e2c4199..5ffedaa 100644 --- a/spiderweb/tests/test_middleware.py +++ b/spiderweb/tests/test_middleware.py @@ -23,8 +23,9 @@ from spiderweb.tests.views_for_tests import ( form_csrf_exempt, form_view_without_csrf, text_view, - unauthorized_view, + unauthorized_view, file_view, ) +from spiderweb.middleware.gzip import GzipMiddleware, CheckValidGzipMinimumLength, CheckValidGzipCompressionLevel def index(request): @@ -349,6 +350,67 @@ def test_unused_post_process_middleware(): assert len(app.middleware) == 0 +class TestGzipMiddleware: + middleware = {"middleware": ["spiderweb.middleware.gzip.GzipMiddleware"]} + + def test_not_enabled_on_small_response(self): + app, environ, start_response = setup( + **self.middleware, + gzip_minimum_response_length=500, + ) + app.add_route("/", text_view) + + environ["HTTP_USER_AGENT"] = "hi" + environ["REMOTE_ADDR"] = "/" + environ["REQUEST_METHOD"] = "GET" + + assert app(environ, start_response) == [bytes("Hi!", DEFAULT_ENCODING)] + assert "Content-Encoding" not in start_response.get_headers() + + def test_changing_minimum_response_length(self): + app, environ, start_response = setup( + **self.middleware, + gzip_minimum_response_length=1, + ) + app.add_route("/", text_view) + + environ["HTTP_ACCEPT_ENCODING"] = "gzip" + environ["HTTP_USER_AGENT"] = "hi" + environ["REMOTE_ADDR"] = "/" + environ["REQUEST_METHOD"] = "GET" + assert str(app(environ, start_response)[0]).startswith("b'\\x1f\\x8b\\x08") + assert "content-encoding" in start_response.get_headers() + + def test_not_enabled_on_error_response(self): + app, environ, start_response = setup( + **self.middleware, + gzip_minimum_response_length=1, + ) + app.add_route("/", unauthorized_view) + + environ["HTTP_ACCEPT_ENCODING"] = "gzip" + environ["HTTP_USER_AGENT"] = "hi" + environ["REMOTE_ADDR"] = "/" + environ["REQUEST_METHOD"] = "GET" + assert app(environ, start_response) == [bytes("Unauthorized", DEFAULT_ENCODING)] + assert "content-encoding" not in start_response.get_headers() + + def test_not_enabled_on_bytes_response(self): + app, environ, start_response = setup( + **self.middleware, + gzip_minimum_response_length=1, + ) + # send a file that's already in bytes form + app.add_route("/", file_view) + + environ["HTTP_ACCEPT_ENCODING"] = "gzip" + environ["HTTP_USER_AGENT"] = "hi" + environ["REMOTE_ADDR"] = "/" + environ["REQUEST_METHOD"] = "GET" + assert app(environ, start_response) == [bytes("hi", DEFAULT_ENCODING)] + assert "content-encoding" not in start_response.get_headers() + + class TestCorsMiddleware: # adapted from: # https://github.com/adamchainz/django-cors-headers/blob/main/tests/test_middleware.py diff --git a/spiderweb/tests/views_for_tests.py b/spiderweb/tests/views_for_tests.py index 3ae8990..dd8b0a3 100644 --- a/spiderweb/tests/views_for_tests.py +++ b/spiderweb/tests/views_for_tests.py @@ -1,7 +1,6 @@ from spiderweb import HttpResponse from spiderweb.decorators import csrf_exempt -from spiderweb.response import JsonResponse, TemplateResponse - +from spiderweb.response import JsonResponse, TemplateResponse, FileResponse EXAMPLE_HTML_FORM = """
@@ -47,3 +46,7 @@ def text_view(request): def unauthorized_view(request): return HttpResponse("Unauthorized", status_code=401) + + +def file_view(request): + return FileResponse("spiderweb/tests/staticfiles/file_for_testing_fileresponse.txt") From ff8f50e44b1d6d03d50e87f98a8710105b5e9005 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Wed, 30 Oct 2024 00:26:10 -0400 Subject: [PATCH 12/15] =?UTF-8?q?=F0=9F=93=9D=20expand=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/middleware/gzip.md | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/docs/middleware/gzip.md b/docs/middleware/gzip.md index 9110692..1199e71 100644 --- a/docs/middleware/gzip.md +++ b/docs/middleware/gzip.md @@ -1,20 +1,39 @@ -# Gzip compress middleware +# gzip compression middleware + +> New in 1.3.1! ```python from spiderweb import SpiderwebRouter app = SpiderwebRouter( - middleware=["spiderweb.middleware.gzip"], + middleware=["spiderweb.middleware.gzip.GzipMiddleware"], ) ``` -When your response is big, you maybe want to reduce traffic between -server and client. -Gzip will help you. This middleware do not cover all possibilities of content compress. Brotli, deflate, zsts or other are out of scope. +If your app is serving large responses, you may want to compress them. We don't (currently) have built-in support for Brotli, deflate, zstd, or other compression methods, but we do support gzip. (Want to add support for other methods? We'd love to see a PR!) -This version only check if gzip method is accepted by client, size of content is greater than 500 bytes. Check if response is not already compressed and response status is between 200 and 300. +The implementation in Spiderweb is simple: it compresses the response body if the client indicates that it is supported. If the client doesn't support gzip, the response is sent uncompressed. Compression happens at the end of the response cycle, so it won't interfere with other middleware. +Error responses and responses with status codes that indicate that the response body should not be sent (like 204, 304, etc.) will not be compressed. Responses with a `Content-Encoding` header already set (e.g. if you're serving pre-compressed files) will be handled the same way. -> [!NOTE] -> Minimal required version is 1.3.1 +The available configuration options are: +## gzip_minimum_response_length + +The minimum size in bytes of a response before it will be compressed. Defaults to `500`. Responses smaller than this will not be compressed. + +```python +app = SpiderwebRouter( + gzip_minimum_response_length=1000 +) +``` + +## gzip_compression_level + +The level of compression to use. Defaults to `6`. This is a number between 0 and 9, where 0 is no compression and 9 is maximum compression. Higher levels will result in smaller files, but will take longer to compress and decompress. Level 6 is a good balance between file size and speed. + +```python +app = SpiderwebRouter( + gzip_compression_level=9 +) +``` From 557cd39c13b75947e6f45dde5241322f04ac45c6 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Wed, 30 Oct 2024 00:31:11 -0400 Subject: [PATCH 13/15] =?UTF-8?q?=E2=9C=85=20add=20tests=20for=20server=20?= =?UTF-8?q?checks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiderweb/tests/test_middleware.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/spiderweb/tests/test_middleware.py b/spiderweb/tests/test_middleware.py index 5ffedaa..3901066 100644 --- a/spiderweb/tests/test_middleware.py +++ b/spiderweb/tests/test_middleware.py @@ -23,9 +23,13 @@ from spiderweb.tests.views_for_tests import ( form_csrf_exempt, form_view_without_csrf, text_view, - unauthorized_view, file_view, + unauthorized_view, + file_view, +) +from spiderweb.middleware.gzip import ( + CheckValidGzipMinimumLength, + CheckValidGzipCompressionLevel, ) -from spiderweb.middleware.gzip import GzipMiddleware, CheckValidGzipMinimumLength, CheckValidGzipCompressionLevel def index(request): @@ -410,6 +414,27 @@ class TestGzipMiddleware: assert app(environ, start_response) == [bytes("hi", DEFAULT_ENCODING)] assert "content-encoding" not in start_response.get_headers() + def test_invalid_response_length(self): + class FakeServer: + gzip_minimum_response_length = "asdf" + with pytest.raises(ConfigError) as e: + CheckValidGzipMinimumLength(server=FakeServer).check() + assert e.value.args[0] == CheckValidGzipMinimumLength.INVALID_GZIP_MINIMUM_LENGTH + + def test_negative_response_length(self): + class FakeServer: + gzip_minimum_response_length = -1 + with pytest.raises(ConfigError) as e: + CheckValidGzipMinimumLength(server=FakeServer).check() + assert e.value.args[0] == CheckValidGzipMinimumLength.INVALID_GZIP_MINIMUM_LENGTH + + def test_bad_compression_level(self): + class FakeServer: + gzip_compression_level = "asdf" + with pytest.raises(ConfigError) as e: + CheckValidGzipCompressionLevel(server=FakeServer).check() + assert e.value.args[0] == CheckValidGzipCompressionLevel.INVALID_GZIP_COMPRESSION_LEVEL + class TestCorsMiddleware: # adapted from: From 991d6be5a3327d88273b6f856877b89e18f52ba5 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Wed, 30 Oct 2024 00:32:02 -0400 Subject: [PATCH 14/15] =?UTF-8?q?=F0=9F=8E=A8=20run=20black?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spiderweb/tests/test_middleware.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/spiderweb/tests/test_middleware.py b/spiderweb/tests/test_middleware.py index 3901066..1914eac 100644 --- a/spiderweb/tests/test_middleware.py +++ b/spiderweb/tests/test_middleware.py @@ -417,23 +417,33 @@ class TestGzipMiddleware: def test_invalid_response_length(self): class FakeServer: gzip_minimum_response_length = "asdf" + with pytest.raises(ConfigError) as e: CheckValidGzipMinimumLength(server=FakeServer).check() - assert e.value.args[0] == CheckValidGzipMinimumLength.INVALID_GZIP_MINIMUM_LENGTH + assert ( + e.value.args[0] == CheckValidGzipMinimumLength.INVALID_GZIP_MINIMUM_LENGTH + ) def test_negative_response_length(self): class FakeServer: gzip_minimum_response_length = -1 + with pytest.raises(ConfigError) as e: CheckValidGzipMinimumLength(server=FakeServer).check() - assert e.value.args[0] == CheckValidGzipMinimumLength.INVALID_GZIP_MINIMUM_LENGTH + assert ( + e.value.args[0] == CheckValidGzipMinimumLength.INVALID_GZIP_MINIMUM_LENGTH + ) def test_bad_compression_level(self): class FakeServer: gzip_compression_level = "asdf" + with pytest.raises(ConfigError) as e: CheckValidGzipCompressionLevel(server=FakeServer).check() - assert e.value.args[0] == CheckValidGzipCompressionLevel.INVALID_GZIP_COMPRESSION_LEVEL + assert ( + e.value.args[0] + == CheckValidGzipCompressionLevel.INVALID_GZIP_COMPRESSION_LEVEL + ) class TestCorsMiddleware: From 7740299ad87e91ecab9ca82f0171509a1820fe5b Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Wed, 30 Oct 2024 00:32:37 -0400 Subject: [PATCH 15/15] =?UTF-8?q?=F0=9F=94=96=20release=201.4.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/middleware/gzip.md | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/middleware/gzip.md b/docs/middleware/gzip.md index 1199e71..08a5cd8 100644 --- a/docs/middleware/gzip.md +++ b/docs/middleware/gzip.md @@ -1,6 +1,6 @@ # gzip compression middleware -> New in 1.3.1! +> New in 1.4.0! ```python from spiderweb import SpiderwebRouter diff --git a/pyproject.toml b/pyproject.toml index fc93003..70f6a75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "spiderweb-framework" -version = "1.3.1" +version = "1.4.0" description = "A small web framework, just big enough for a spider." authors = ["Joe Kaufeld "] readme = "README.md"