fix(plugin): bundle Python dependencies for standalone deployment

This commit is contained in:
2026-08-26 11:53:59 +02:00
parent e4a93cdc12
commit 0055397ed5
319 changed files with 95549 additions and 18 deletions
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: MIT
from ._ffi import ffi, lib
__all__ = ["ffi", "lib"]
Binary file not shown.
@@ -0,0 +1,228 @@
# SPDX-License-Identifier: MIT
import os
import platform
from pathlib import Path
from cffi import FFI
use_system_argon2 = os.environ.get("ARGON2_CFFI_USE_SYSTEM", "0") == "1"
use_sse2 = os.environ.get("ARGON2_CFFI_USE_SSE2", None)
windows = platform.system() == "Windows"
# Try to detect cross-compilation.
def _get_target_platform(arch_flags, default):
flags = [f for f in arch_flags.split(" ") if f.strip() != ""]
try:
pos = flags.index("-arch")
return flags[pos + 1].lower()
except ValueError:
pass
return default
target_platform = _get_target_platform(
os.environ.get("ARCHFLAGS", ""), platform.machine()
)
if use_sse2 == "1":
optimized = True
elif use_sse2 == "0":
optimized = False
else:
# Optimized version requires SSE2 extensions. They have been around since
# 2001 so we try to compile it on every recent-ish x86.
optimized = target_platform in ("i686", "x86", "x86_64", "AMD64")
ffi = FFI()
if use_system_argon2:
ffi.set_source(
"_ffi",
"#include <argon2.h>",
libraries=["argon2"],
)
else:
lib_base = Path("extras") / "libargon2" / "src"
ffi.set_source(
"_ffi",
"#include <argon2.h>",
extra_compile_args=["-msse2"] if (optimized and not windows) else None,
include_dirs=[os.path.join("extras", "libargon2", "include")],
sources=[
str(lib_base / path)
for path in [
"argon2.c",
Path("blake2") / "blake2b.c",
"core.c",
"encoding.c",
"opt.c" if optimized else "ref.c",
"thread.c",
]
],
)
ffi.cdef(
"""\
typedef enum Argon2_type {
Argon2_d = ...,
Argon2_i = ...,
Argon2_id = ...,
} argon2_type;
typedef enum Argon2_version {
ARGON2_VERSION_10 = ...,
ARGON2_VERSION_13 = ...,
ARGON2_VERSION_NUMBER = ...
} argon2_version;
int argon2_hash(const uint32_t t_cost, const uint32_t m_cost,
const uint32_t parallelism, const void *pwd,
const size_t pwdlen, const void *salt,
const size_t saltlen, void *hash,
const size_t hashlen, char *encoded,
const size_t encodedlen, argon2_type type,
const uint32_t version);
int argon2_verify(const char *encoded, const void *pwd,
const size_t pwdlen, argon2_type type);
const char *argon2_error_message(int error_code);
typedef int (*allocate_fptr)(uint8_t **memory, size_t bytes_to_allocate);
typedef void (*deallocate_fptr)(uint8_t *memory, size_t bytes_to_allocate);
typedef struct Argon2_Context {
uint8_t *out; /* output array */
uint32_t outlen; /* digest length */
uint8_t *pwd; /* password array */
uint32_t pwdlen; /* password length */
uint8_t *salt; /* salt array */
uint32_t saltlen; /* salt length */
uint8_t *secret; /* key array */
uint32_t secretlen; /* key length */
uint8_t *ad; /* associated data array */
uint32_t adlen; /* associated data length */
uint32_t t_cost; /* number of passes */
uint32_t m_cost; /* amount of memory requested (KB) */
uint32_t lanes; /* number of lanes */
uint32_t threads; /* maximum number of threads */
uint32_t version; /* version number */
allocate_fptr allocate_cbk; /* pointer to memory allocator */
deallocate_fptr free_cbk; /* pointer to memory deallocator */
uint32_t flags; /* array of bool options */
} argon2_context;
int argon2_ctx(argon2_context *context, argon2_type type);
/* Error codes */
typedef enum Argon2_ErrorCodes {
ARGON2_OK = ...,
ARGON2_OUTPUT_PTR_NULL = ...,
ARGON2_OUTPUT_TOO_SHORT = ...,
ARGON2_OUTPUT_TOO_LONG = ...,
ARGON2_PWD_TOO_SHORT = ...,
ARGON2_PWD_TOO_LONG = ...,
ARGON2_SALT_TOO_SHORT = ...,
ARGON2_SALT_TOO_LONG = ...,
ARGON2_AD_TOO_SHORT = ...,
ARGON2_AD_TOO_LONG = ...,
ARGON2_SECRET_TOO_SHORT = ...,
ARGON2_SECRET_TOO_LONG = ...,
ARGON2_TIME_TOO_SMALL = ...,
ARGON2_TIME_TOO_LARGE = ...,
ARGON2_MEMORY_TOO_LITTLE = ...,
ARGON2_MEMORY_TOO_MUCH = ...,
ARGON2_LANES_TOO_FEW = ...,
ARGON2_LANES_TOO_MANY = ...,
ARGON2_PWD_PTR_MISMATCH = ..., /* NULL ptr with non-zero length */
ARGON2_SALT_PTR_MISMATCH = ..., /* NULL ptr with non-zero length */
ARGON2_SECRET_PTR_MISMATCH = ..., /* NULL ptr with non-zero length */
ARGON2_AD_PTR_MISMATCH = ..., /* NULL ptr with non-zero length */
ARGON2_MEMORY_ALLOCATION_ERROR = ...,
ARGON2_FREE_MEMORY_CBK_NULL = ...,
ARGON2_ALLOCATE_MEMORY_CBK_NULL = ...,
ARGON2_INCORRECT_PARAMETER = ...,
ARGON2_INCORRECT_TYPE = ...,
ARGON2_OUT_PTR_MISMATCH = ...,
ARGON2_THREADS_TOO_FEW = ...,
ARGON2_THREADS_TOO_MANY = ...,
ARGON2_MISSING_ARGS = ...,
ARGON2_ENCODING_FAIL = ...,
ARGON2_DECODING_FAIL = ...,
ARGON2_THREAD_FAIL = ...,
ARGON2_DECODING_LENGTH_FAIL= ...,
ARGON2_VERIFY_MISMATCH = ...,
} argon2_error_codes;
#define ARGON2_FLAG_CLEAR_PASSWORD ...
#define ARGON2_FLAG_CLEAR_SECRET ...
#define ARGON2_DEFAULT_FLAGS ...
#define ARGON2_MIN_LANES ...
#define ARGON2_MAX_LANES ...
#define ARGON2_MIN_THREADS ...
#define ARGON2_MAX_THREADS ...
#define ARGON2_SYNC_POINTS ...
#define ARGON2_MIN_OUTLEN ...
#define ARGON2_MAX_OUTLEN ...
#define ARGON2_MIN_MEMORY ...
#define ARGON2_MAX_MEMORY_BITS ...
#define ARGON2_MAX_MEMORY ...
#define ARGON2_MIN_TIME ...
#define ARGON2_MAX_TIME ...
#define ARGON2_MIN_PWD_LENGTH ...
#define ARGON2_MAX_PWD_LENGTH ...
#define ARGON2_MIN_AD_LENGTH ...
#define ARGON2_MAX_AD_LENGTH ...
#define ARGON2_MIN_SALT_LENGTH ...
#define ARGON2_MAX_SALT_LENGTH ...
#define ARGON2_MIN_SECRET ...
#define ARGON2_MAX_SECRET ...
uint32_t argon2_encodedlen(uint32_t t_cost, uint32_t m_cost,
uint32_t parallelism, uint32_t saltlen,
uint32_t hashlen, argon2_type type);
"""
)
if __name__ == "__main__":
ffi.compile()
Binary file not shown.
+14
View File
@@ -0,0 +1,14 @@
__version__ = "2.7.1"
from .impl import start_connection
from .types import AddrInfoType, SocketFactoryType
from .utils import addr_to_addr_infos, pop_addr_infos_interleave, remove_addr_infos
__all__ = (
"AddrInfoType",
"SocketFactoryType",
"addr_to_addr_infos",
"pop_addr_infos_interleave",
"remove_addr_infos",
"start_connection",
)
+197
View File
@@ -0,0 +1,197 @@
import asyncio
import contextlib
from collections.abc import Awaitable, Callable, Iterable
from typing import (
TYPE_CHECKING,
Any,
TypeVar,
)
_T = TypeVar("_T")
RE_RAISE_EXCEPTIONS = (SystemExit, KeyboardInterrupt)
def _set_result(wait_next: "asyncio.Future[None]") -> None:
"""Set the result of a future if it is not already done."""
if not wait_next.done():
wait_next.set_result(None)
async def _wait_one(
futures: "Iterable[asyncio.Future[Any]]",
loop: asyncio.AbstractEventLoop,
) -> _T:
"""Wait for the first future to complete."""
wait_next = loop.create_future()
def _on_completion(fut: "asyncio.Future[Any]") -> None:
if not wait_next.done():
wait_next.set_result(fut)
for f in futures:
f.add_done_callback(_on_completion)
try:
return await wait_next
finally:
for f in futures:
f.remove_done_callback(_on_completion)
async def staggered_race(
coro_fns: Iterable[Callable[[], Awaitable[_T]]],
delay: float | None,
*,
loop: asyncio.AbstractEventLoop | None = None,
) -> tuple[_T | None, int | None, list[BaseException | None]]:
"""
Run coroutines with staggered start times and take the first to finish.
This method takes an iterable of coroutine functions. The first one is
started immediately. From then on, whenever the immediately preceding one
fails (raises an exception), or when *delay* seconds has passed, the next
coroutine is started. This continues until one of the coroutines complete
successfully, in which case all others are cancelled, or until all
coroutines fail.
The coroutines provided should be well-behaved in the following way:
* They should only ``return`` if completed successfully.
* They should always raise an exception if they did not complete
successfully. In particular, if they handle cancellation, they should
probably reraise, like this::
try:
# do work
except asyncio.CancelledError:
# undo partially completed work
raise
Args:
----
coro_fns: an iterable of coroutine functions, i.e. callables that
return a coroutine object when called. Use ``functools.partial`` or
lambdas to pass arguments.
delay: amount of time, in seconds, between starting coroutines. If
``None``, the coroutines will run sequentially.
loop: the event loop to use. If ``None``, the running loop is used.
Returns:
-------
tuple *(winner_result, winner_index, exceptions)* where
- *winner_result*: the result of the winning coroutine, or ``None``
if no coroutines won.
- *winner_index*: the index of the winning coroutine in
``coro_fns``, or ``None`` if no coroutines won. If the winning
coroutine may return None on success, *winner_index* can be used
to definitively determine whether any coroutine won.
- *exceptions*: list of exceptions returned by the coroutines.
``len(exceptions)`` is equal to the number of coroutines actually
started, and the order is the same as in ``coro_fns``. The winning
coroutine's entry is ``None``.
"""
loop = loop or asyncio.get_running_loop()
exceptions: list[BaseException | None] = []
tasks: set[asyncio.Task[tuple[_T, int] | None]] = set()
async def run_one_coro(
coro_fn: Callable[[], Awaitable[_T]],
this_index: int,
start_next: "asyncio.Future[None]",
) -> tuple[_T, int] | None:
"""
Run a single coroutine.
If the coroutine fails, set the exception in the exceptions list and
start the next coroutine by setting the result of the start_next.
If the coroutine succeeds, return the result and the index of the
coroutine in the coro_fns list.
If SystemExit or KeyboardInterrupt is raised, re-raise it.
"""
try:
result = await coro_fn()
except RE_RAISE_EXCEPTIONS:
raise
except BaseException as e:
exceptions[this_index] = e
_set_result(start_next) # Kickstart the next coroutine
return None
return result, this_index
start_next_timer: asyncio.TimerHandle | None = None
start_next: asyncio.Future[None] | None
task: asyncio.Task[tuple[_T, int] | None]
done: asyncio.Future[None] | asyncio.Task[tuple[_T, int] | None]
coro_iter = iter(coro_fns)
this_index = -1
try:
while True:
if coro_fn := next(coro_iter, None):
this_index += 1
exceptions.append(None)
start_next = loop.create_future()
task = loop.create_task(run_one_coro(coro_fn, this_index, start_next))
tasks.add(task)
start_next_timer = (
loop.call_later(delay, _set_result, start_next) if delay else None
)
elif not tasks:
# We exhausted the coro_fns list and no tasks are running
# so we have no winner and all coroutines failed.
break
while tasks or start_next:
done = await _wait_one(
(*tasks, start_next) if start_next else tasks, loop
)
if done is start_next:
# The current task has failed or the timer has expired
# so we need to start the next task.
start_next = None
if start_next_timer:
start_next_timer.cancel()
start_next_timer = None
# Break out of the task waiting loop to start the next
# task.
break
if TYPE_CHECKING:
assert isinstance(done, asyncio.Task)
tasks.remove(done)
if winner := done.result():
return *winner, exceptions
finally:
# We either have:
# - a winner
# - all tasks failed
# - a KeyboardInterrupt or SystemExit.
#
# If the timer is still running, cancel it.
#
if start_next_timer:
start_next_timer.cancel()
#
# If there are any tasks left, cancel them and than
# wait them so they fill the exceptions list.
#
for task in tasks:
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
return None, None, exceptions
+247
View File
@@ -0,0 +1,247 @@
"""Base implementation."""
import asyncio
import contextlib
import functools
import itertools
import socket
from collections import defaultdict
from collections.abc import Sequence
from . import _staggered
from .types import AddrInfoType, SocketFactoryType
async def start_connection(
addr_infos: Sequence[AddrInfoType],
*,
local_addr_infos: Sequence[AddrInfoType] | None = None,
happy_eyeballs_delay: float | None = None,
interleave: int | None = None,
loop: asyncio.AbstractEventLoop | None = None,
socket_factory: SocketFactoryType | None = None,
) -> socket.socket:
"""
Connect to a TCP server.
Create a socket connection to a specified destination. The
destination is specified as a list of AddrInfoType tuples as
returned from getaddrinfo().
The arguments are, in order:
* ``family``: the address family, e.g. ``socket.AF_INET`` or
``socket.AF_INET6``.
* ``type``: the socket type, e.g. ``socket.SOCK_STREAM`` or
``socket.SOCK_DGRAM``.
* ``proto``: the protocol, e.g. ``socket.IPPROTO_TCP`` or
``socket.IPPROTO_UDP``.
* ``canonname``: the canonical name of the address, e.g.
``"www.python.org"``.
* ``sockaddr``: the socket address
This method is a coroutine which will try to establish the connection
in the background. When successful, the coroutine returns a
socket.
The expected use case is to use this method in conjunction with
loop.create_connection() to establish a connection to a server::
socket = await start_connection(addr_infos)
transport, protocol = await loop.create_connection(
MyProtocol, sock=socket, ...)
"""
if not addr_infos:
raise ValueError("addr_infos must not be empty")
current_loop = loop or asyncio.get_running_loop()
single_addr_info = len(addr_infos) == 1
if happy_eyeballs_delay is not None and interleave is None:
# If using happy eyeballs, default to interleave addresses by family
interleave = 1
if interleave and not single_addr_info:
addr_infos = _interleave_addrinfos(addr_infos, interleave)
sock: socket.socket | None = None
# uvloop can raise RuntimeError instead of OSError
exceptions: list[list[OSError | RuntimeError]] = []
if happy_eyeballs_delay is None or single_addr_info:
# not using happy eyeballs
for addrinfo in addr_infos:
try:
sock = await _connect_sock(
current_loop,
exceptions,
addrinfo,
local_addr_infos,
None,
socket_factory,
)
break
except (RuntimeError, OSError):
continue
else: # using happy eyeballs
open_sockets: set[socket.socket] = set()
try:
sock, _, _ = await _staggered.staggered_race(
(
functools.partial(
_connect_sock,
current_loop,
exceptions,
addrinfo,
local_addr_infos,
open_sockets,
socket_factory,
)
for addrinfo in addr_infos
),
happy_eyeballs_delay,
)
finally:
# If we have a winner, staggered_race will
# cancel the other tasks, however there is a
# small race window where any of the other tasks
# can be done before they are cancelled which
# will leave the socket open. To avoid this problem
# we pass a set to _connect_sock to keep track of
# the open sockets and close them here if there
# are any "runner up" sockets.
for s in open_sockets:
if s is not sock:
with contextlib.suppress(OSError):
s.close()
open_sockets = None # type: ignore[assignment]
if sock is None:
all_exceptions = [exc for sub in exceptions for exc in sub]
try:
first_exception = all_exceptions[0]
if len(all_exceptions) == 1:
raise first_exception
else:
# If they all have the same str(), raise one.
model = str(first_exception)
if all(str(exc) == model for exc in all_exceptions):
raise first_exception
# Raise a combined exception so the user can see all
# the various error messages.
msg = "Multiple exceptions: {}".format(
", ".join(str(exc) for exc in all_exceptions)
)
# If the errno is the same for all exceptions, raise
# an OSError with that errno.
if isinstance(first_exception, OSError):
first_errno = first_exception.errno
if all(
isinstance(exc, OSError) and exc.errno == first_errno
for exc in all_exceptions
):
raise OSError(first_errno, msg)
elif isinstance(first_exception, RuntimeError) and all(
isinstance(exc, RuntimeError) for exc in all_exceptions
):
raise RuntimeError(msg)
# We have a mix of OSError and RuntimeError
# so we have to pick which one to raise.
# and we raise OSError for compatibility
raise OSError(msg)
finally:
all_exceptions = None # type: ignore[assignment]
exceptions = None # type: ignore[assignment]
return sock
async def _connect_sock(
loop: asyncio.AbstractEventLoop,
exceptions: list[list[OSError | RuntimeError]],
addr_info: AddrInfoType,
local_addr_infos: Sequence[AddrInfoType] | None = None,
open_sockets: set[socket.socket] | None = None,
socket_factory: SocketFactoryType | None = None,
) -> socket.socket:
"""
Create, bind and connect one socket.
If open_sockets is passed, add the socket to the set of open sockets.
Any failure caught here will remove the socket from the set and close it.
Callers can use this set to close any sockets that are not the winner
of all staggered tasks in the result there are runner up sockets aka
multiple winners.
"""
my_exceptions: list[OSError | RuntimeError] = []
exceptions.append(my_exceptions)
family, type_, proto, _, address = addr_info
sock = None
try:
if socket_factory is not None:
sock = socket_factory(addr_info)
else:
sock = socket.socket(family=family, type=type_, proto=proto)
if open_sockets is not None:
open_sockets.add(sock)
sock.setblocking(False)
if local_addr_infos is not None:
for lfamily, _, _, _, laddr in local_addr_infos:
# skip local addresses of different family
if lfamily != family:
continue
try:
sock.bind(laddr)
break
except OSError as exc:
msg = (
f"error while attempting to bind on "
f"address {laddr!r}: "
f"{(exc.strerror or '').lower()}"
)
exc = OSError(exc.errno, msg)
my_exceptions.append(exc)
else: # all bind attempts failed
if my_exceptions:
raise my_exceptions.pop()
else:
raise OSError(f"no matching local address with {family=} found")
await loop.sock_connect(sock, address)
return sock
except BaseException as exc:
if isinstance(exc, (RuntimeError, OSError)):
my_exceptions.append(exc)
if sock is not None:
if open_sockets is not None:
open_sockets.remove(sock)
try:
sock.close()
except OSError as e:
my_exceptions.append(e)
raise
raise
finally:
exceptions = my_exceptions = None # type: ignore[assignment]
def _interleave_addrinfos(
addrinfos: Sequence[AddrInfoType], first_address_family_count: int = 1
) -> list[AddrInfoType]:
"""Interleave list of addrinfo tuples by family."""
# Group addresses by family
addrinfos_by_family: defaultdict[int, list[AddrInfoType]] = defaultdict(list)
for addr in addrinfos:
addrinfos_by_family[addr[0]].append(addr)
addrinfos_lists = list(addrinfos_by_family.values())
reordered: list[AddrInfoType] = []
if first_address_family_count > 1:
reordered.extend(addrinfos_lists[0][: first_address_family_count - 1])
del addrinfos_lists[0][: first_address_family_count - 1]
reordered.extend(
a
for a in itertools.chain.from_iterable(itertools.zip_longest(*addrinfos_lists))
if a is not None
)
return reordered
+14
View File
@@ -0,0 +1,14 @@
"""Types for aiohappyeyeballs."""
import socket
from collections.abc import Callable
AddrInfoType = tuple[
int | socket.AddressFamily,
int | socket.SocketKind,
int,
str,
tuple, # type: ignore[type-arg]
]
SocketFactoryType = Callable[[AddrInfoType], socket.socket]
+82
View File
@@ -0,0 +1,82 @@
"""Utility functions for aiohappyeyeballs."""
import ipaddress
import socket
from .types import AddrInfoType
def addr_to_addr_infos(
addr: tuple[str, int, int, int] | tuple[str, int, int] | tuple[str, int] | None,
) -> list[AddrInfoType] | None:
"""Convert an address tuple to a list of addr_info tuples."""
if addr is None:
return None
host = addr[0]
port = addr[1]
is_ipv6 = ":" in host
if is_ipv6:
flowinfo = 0
scopeid = 0
addr_len = len(addr)
if addr_len >= 4:
scopeid = addr[3] # type: ignore[misc]
if addr_len >= 3:
flowinfo = addr[2] # type: ignore[misc]
addr = (host, port, flowinfo, scopeid)
family = socket.AF_INET6
else:
addr = (host, port)
family = socket.AF_INET
return [(family, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", addr)]
def pop_addr_infos_interleave(
addr_infos: list[AddrInfoType], interleave: int | None = None
) -> None:
"""
Pop addr_info from the list of addr_infos by family up to interleave times.
The interleave parameter is used to know how many addr_infos for
each family should be popped of the top of the list.
"""
if interleave is None:
interleave = 1
seen: dict[int, int] = {}
kept: list[AddrInfoType] = []
for addr_info in addr_infos:
family = addr_info[0]
count = seen.get(family, 0)
if count >= interleave:
kept.append(addr_info)
seen[family] = count + 1
addr_infos[:] = kept
def _addr_tuple_to_ip_address(
addr: tuple[str, int] | tuple[str, int, int, int],
) -> tuple[ipaddress.IPv4Address, int] | tuple[ipaddress.IPv6Address, int, int, int]:
"""Convert an address tuple to an IPv4Address."""
return (ipaddress.ip_address(addr[0]), *addr[1:])
def remove_addr_infos(
addr_infos: list[AddrInfoType],
addr: tuple[str, int] | tuple[str, int, int, int],
) -> None:
"""
Remove an address from the list of addr_infos.
The addr value is typically the return value of
sock.getpeername().
"""
kept = [ai for ai in addr_infos if ai[-1] != addr]
if len(kept) == len(addr_infos):
# Slow path in case addr is formatted differently
match_addr = _addr_tuple_to_ip_address(addr)
kept = [
ai for ai in addr_infos if _addr_tuple_to_ip_address(ai[-1]) != match_addr
]
if len(kept) == len(addr_infos):
raise ValueError(f"Address {addr} not found in addr_infos")
addr_infos[:] = kept
+279
View File
@@ -0,0 +1,279 @@
__version__ = "3.14.3"
from typing import TYPE_CHECKING
from . import hdrs as hdrs
from .client import (
BaseConnector,
ClientConnectionError,
ClientConnectionResetError,
ClientConnectorCertificateError,
ClientConnectorDNSError,
ClientConnectorError,
ClientConnectorSSLError,
ClientError,
ClientHttpProxyError,
ClientOSError,
ClientPayloadError,
ClientProxyConnectionError,
ClientRequest,
ClientResponse,
ClientResponseError,
ClientSession,
ClientSSLError,
ClientTimeout,
ClientWebSocketResponse,
ClientWSTimeout,
ConnectionTimeoutError,
ContentTypeError,
Fingerprint,
InvalidURL,
InvalidUrlClientError,
InvalidUrlRedirectClientError,
NamedPipeConnector,
NonHttpUrlClientError,
NonHttpUrlRedirectClientError,
RedirectClientError,
RequestInfo,
ServerConnectionError,
ServerDisconnectedError,
ServerFingerprintMismatch,
ServerTimeoutError,
SocketTimeoutError,
TCPConnector,
TooManyRedirects,
UnixConnector,
WSMessageTypeError,
WSServerHandshakeError,
request,
)
from .client_middleware_digest_auth import DigestAuthMiddleware
from .client_middlewares import ClientHandlerType, ClientMiddlewareType
from .compression_utils import set_zlib_backend
from .connector import (
AddrInfoType as AddrInfoType,
SocketFactoryType as SocketFactoryType,
)
from .cookiejar import CookieJar as CookieJar, DummyCookieJar as DummyCookieJar
from .formdata import FormData as FormData
from .helpers import BasicAuth, ChainMapProxy, ETag, encode_basic_auth
from .http import (
HttpVersion as HttpVersion,
HttpVersion10 as HttpVersion10,
HttpVersion11 as HttpVersion11,
WebSocketError as WebSocketError,
WSCloseCode as WSCloseCode,
WSMessage as WSMessage,
WSMsgType as WSMsgType,
)
from .multipart import (
BadContentDispositionHeader as BadContentDispositionHeader,
BadContentDispositionParam as BadContentDispositionParam,
BodyPartReader as BodyPartReader,
MultipartReader as MultipartReader,
MultipartWriter as MultipartWriter,
content_disposition_filename as content_disposition_filename,
parse_content_disposition as parse_content_disposition,
)
from .payload import (
PAYLOAD_REGISTRY as PAYLOAD_REGISTRY,
AsyncIterablePayload as AsyncIterablePayload,
BufferedReaderPayload as BufferedReaderPayload,
BytesIOPayload as BytesIOPayload,
BytesPayload as BytesPayload,
IOBasePayload as IOBasePayload,
JsonPayload as JsonPayload,
Payload as Payload,
StringIOPayload as StringIOPayload,
StringPayload as StringPayload,
TextIOPayload as TextIOPayload,
get_payload as get_payload,
payload_type as payload_type,
)
from .payload_streamer import streamer as streamer
from .resolver import (
AsyncResolver as AsyncResolver,
DefaultResolver as DefaultResolver,
ThreadedResolver as ThreadedResolver,
)
from .streams import (
EMPTY_PAYLOAD as EMPTY_PAYLOAD,
DataQueue as DataQueue,
EofStream as EofStream,
FlowControlDataQueue as FlowControlDataQueue,
StreamReader as StreamReader,
)
from .tracing import (
TraceConfig as TraceConfig,
TraceConnectionCreateEndParams as TraceConnectionCreateEndParams,
TraceConnectionCreateStartParams as TraceConnectionCreateStartParams,
TraceConnectionQueuedEndParams as TraceConnectionQueuedEndParams,
TraceConnectionQueuedStartParams as TraceConnectionQueuedStartParams,
TraceConnectionReuseconnParams as TraceConnectionReuseconnParams,
TraceDnsCacheHitParams as TraceDnsCacheHitParams,
TraceDnsCacheMissParams as TraceDnsCacheMissParams,
TraceDnsResolveHostEndParams as TraceDnsResolveHostEndParams,
TraceDnsResolveHostStartParams as TraceDnsResolveHostStartParams,
TraceRequestChunkSentParams as TraceRequestChunkSentParams,
TraceRequestEndParams as TraceRequestEndParams,
TraceRequestExceptionParams as TraceRequestExceptionParams,
TraceRequestHeadersSentParams as TraceRequestHeadersSentParams,
TraceRequestRedirectParams as TraceRequestRedirectParams,
TraceRequestStartParams as TraceRequestStartParams,
TraceResponseChunkReceivedParams as TraceResponseChunkReceivedParams,
)
if TYPE_CHECKING:
# At runtime these are lazy-loaded at the bottom of the file.
from .worker import (
GunicornUVLoopWebWorker as GunicornUVLoopWebWorker,
GunicornWebWorker as GunicornWebWorker,
)
__all__: tuple[str, ...] = (
"hdrs",
# client
"AddrInfoType",
"BaseConnector",
"ClientConnectionError",
"ClientConnectionResetError",
"ClientConnectorCertificateError",
"ClientConnectorDNSError",
"ClientConnectorError",
"ClientConnectorSSLError",
"ClientError",
"ClientHttpProxyError",
"ClientOSError",
"ClientPayloadError",
"ClientProxyConnectionError",
"ClientResponse",
"ClientRequest",
"ClientResponseError",
"ClientSSLError",
"ClientSession",
"ClientTimeout",
"ClientWebSocketResponse",
"ClientWSTimeout",
"ConnectionTimeoutError",
"ContentTypeError",
"Fingerprint",
"FlowControlDataQueue",
"InvalidURL",
"InvalidUrlClientError",
"InvalidUrlRedirectClientError",
"NonHttpUrlClientError",
"NonHttpUrlRedirectClientError",
"RedirectClientError",
"RequestInfo",
"ServerConnectionError",
"ServerDisconnectedError",
"ServerFingerprintMismatch",
"ServerTimeoutError",
"SocketFactoryType",
"SocketTimeoutError",
"TCPConnector",
"TooManyRedirects",
"UnixConnector",
"NamedPipeConnector",
"WSServerHandshakeError",
"request",
# client_middleware
"ClientMiddlewareType",
"ClientHandlerType",
# cookiejar
"CookieJar",
"DummyCookieJar",
# formdata
"FormData",
# helpers
"BasicAuth",
"ChainMapProxy",
"DigestAuthMiddleware",
"ETag",
"encode_basic_auth",
"set_zlib_backend",
# http
"HttpVersion",
"HttpVersion10",
"HttpVersion11",
"WSMsgType",
"WSCloseCode",
"WSMessage",
"WebSocketError",
# multipart
"BadContentDispositionHeader",
"BadContentDispositionParam",
"BodyPartReader",
"MultipartReader",
"MultipartWriter",
"content_disposition_filename",
"parse_content_disposition",
# payload
"AsyncIterablePayload",
"BufferedReaderPayload",
"BytesIOPayload",
"BytesPayload",
"IOBasePayload",
"JsonPayload",
"PAYLOAD_REGISTRY",
"Payload",
"StringIOPayload",
"StringPayload",
"TextIOPayload",
"get_payload",
"payload_type",
# payload_streamer
"streamer",
# resolver
"AsyncResolver",
"DefaultResolver",
"ThreadedResolver",
# streams
"DataQueue",
"EMPTY_PAYLOAD",
"EofStream",
"StreamReader",
# tracing
"TraceConfig",
"TraceConnectionCreateEndParams",
"TraceConnectionCreateStartParams",
"TraceConnectionQueuedEndParams",
"TraceConnectionQueuedStartParams",
"TraceConnectionReuseconnParams",
"TraceDnsCacheHitParams",
"TraceDnsCacheMissParams",
"TraceDnsResolveHostEndParams",
"TraceDnsResolveHostStartParams",
"TraceRequestChunkSentParams",
"TraceRequestEndParams",
"TraceRequestExceptionParams",
"TraceRequestHeadersSentParams",
"TraceRequestRedirectParams",
"TraceRequestStartParams",
"TraceResponseChunkReceivedParams",
# workers (imported lazily with __getattr__)
"GunicornUVLoopWebWorker",
"GunicornWebWorker",
"WSMessageTypeError",
)
def __dir__() -> tuple[str, ...]:
return __all__ + ("__doc__",)
def __getattr__(name: str) -> object:
global GunicornUVLoopWebWorker, GunicornWebWorker
# Importing gunicorn takes a long time (>100ms), so only import if actually needed.
if name in ("GunicornUVLoopWebWorker", "GunicornWebWorker"):
try:
from .worker import GunicornUVLoopWebWorker as guv, GunicornWebWorker as gw
except ImportError:
return None
GunicornUVLoopWebWorker = guv # type: ignore[misc]
GunicornWebWorker = gw # type: ignore[misc]
return guv if name == "GunicornUVLoopWebWorker" else gw
raise AttributeError(f"module {__name__} has no attribute {name}")
+361
View File
@@ -0,0 +1,361 @@
"""
Internal cookie handling helpers.
This module contains internal utilities for cookie parsing and manipulation.
These are not part of the public API and may change without notice.
"""
import re
from collections.abc import Sequence
from http.cookies import CookieError, Morsel
from typing import cast
from .log import internal_logger
__all__ = (
"parse_set_cookie_headers",
"parse_cookie_header",
"preserve_morsel_with_coded_value",
)
# Cookie parsing constants
# Allow more characters in cookie names to handle real-world cookies
# that don't strictly follow RFC standards (fixes #2683)
# RFC 6265 defines cookie-name token as per RFC 2616 Section 2.2,
# but many servers send cookies with characters like {} [] () etc.
# This makes the cookie parser more tolerant of real-world cookies
# while still providing some validation to catch obviously malformed names.
_COOKIE_NAME_RE = re.compile(r"^[!#$%&\'()*+\-./0-9:<=>?@A-Z\[\]^_`a-z{|}~]+$")
_COOKIE_KNOWN_ATTRS = frozenset( # AKA Morsel._reserved
(
"path",
"domain",
"max-age",
"expires",
"secure",
"httponly",
"samesite",
"partitioned",
"version",
"comment",
)
)
_COOKIE_BOOL_ATTRS = frozenset( # AKA Morsel._flags
("secure", "httponly", "partitioned")
)
# SimpleCookie's pattern for parsing cookies with relaxed validation
# Based on http.cookies pattern but extended to allow more characters in cookie names
# to handle real-world cookies (fixes #2683)
_COOKIE_PATTERN = re.compile(
r"""
\s* # Optional whitespace at start of cookie
(?P<key> # Start of group 'key'
# aiohttp has extended to include [] for compatibility with real-world cookies
[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\[\]]+ # Any word of at least one letter
) # End of group 'key'
( # Optional group: there may not be a value.
\s*=\s* # Equal Sign
(?P<val> # Start of group 'val'
"(?:[^\\"]|\\.)*" # Any double-quoted string (properly closed)
| # or
"[^";]* # Unmatched opening quote (differs from SimpleCookie - issue #7993)
| # or
# Special case for "expires" attr - RFC 822, RFC 850, RFC 1036, RFC 1123
(\w{3,6}day|\w{3}),\s # Day of the week or abbreviated day (with comma)
[\w\d\s-]{9,11}\s[\d:]{8}\s # Date and time in specific format
(GMT|[+-]\d{4}) # Timezone: GMT or RFC 2822 offset like -0000, +0100
# NOTE: RFC 2822 timezone support is an aiohttp extension
# for issue #4493 - SimpleCookie does NOT support this
| # or
# ANSI C asctime() format: "Wed Jun 9 10:18:14 2021"
# NOTE: This is an aiohttp extension for issue #4327 - SimpleCookie does NOT support this format
\w{3}\s+\w{3}\s+[\s\d]\d\s+\d{2}:\d{2}:\d{2}\s+\d{4}
| # or
[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=\[\]]* # Any word or empty string
) # End of group 'val'
)? # End of optional value group
\s* # Any number of spaces.
(\s+|;|$) # Ending either at space, semicolon, or EOS.
""",
re.VERBOSE | re.ASCII,
)
def preserve_morsel_with_coded_value(cookie: Morsel[str]) -> Morsel[str]:
"""
Preserve a Morsel's coded_value exactly as received from the server.
This function ensures that cookie encoding is preserved exactly as sent by
the server, which is critical for compatibility with old servers that have
strict requirements about cookie formats.
This addresses the issue described in https://github.com/aio-libs/aiohttp/pull/1453
where Python's SimpleCookie would re-encode cookies, breaking authentication
with certain servers.
Args:
cookie: A Morsel object from SimpleCookie
Returns:
A Morsel object with preserved coded_value
"""
mrsl_val = cast("Morsel[str]", cookie.get(cookie.key, Morsel()))
# We use __setstate__ instead of the public set() API because it allows us to
# bypass validation and set already validated state. This is more stable than
# setting protected attributes directly and unlikely to change since it would
# break pickling.
try:
mrsl_val.__setstate__( # type: ignore[attr-defined]
{
"key": cookie.key,
"value": cookie.value,
"coded_value": cookie.coded_value,
}
)
except CookieError:
return cookie
return mrsl_val
_unquote_sub = re.compile(r"\\(?:([0-3][0-7][0-7])|(.))").sub
def _unquote_replace(m: re.Match[str]) -> str:
"""
Replace function for _unquote_sub regex substitution.
Handles escaped characters in cookie values:
- Octal sequences are converted to their character representation
- Other escaped characters are unescaped by removing the backslash
"""
if m[1]:
return chr(int(m[1], 8))
return m[2]
def _unquote(value: str) -> str:
"""
Unquote a cookie value.
Vendored from http.cookies._unquote to ensure compatibility.
Note: The original implementation checked for None, but we've removed
that check since all callers already ensure the value is not None.
"""
# If there aren't any doublequotes,
# then there can't be any special characters. See RFC 2109.
if len(value) < 2:
return value
if value[0] != '"' or value[-1] != '"':
return value
# We have to assume that we must decode this string.
# Down to work.
# Remove the "s
value = value[1:-1]
# Check for special sequences. Examples:
# \012 --> \n
# \" --> "
#
return _unquote_sub(_unquote_replace, value)
def parse_cookie_header(header: str) -> list[tuple[str, Morsel[str]]]:
"""
Parse a Cookie header according to RFC 6265 Section 5.4.
Cookie headers contain only name-value pairs separated by semicolons.
There are no attributes in Cookie headers - even names that match
attribute names (like 'path' or 'secure') should be treated as cookies.
This parser uses the same regex-based approach as parse_set_cookie_headers
to properly handle quoted values that may contain semicolons. When the
regex fails to match a malformed cookie, it falls back to simple parsing
to ensure subsequent cookies are not lost
https://github.com/aio-libs/aiohttp/issues/11632
Args:
header: The Cookie header value to parse
Returns:
List of (name, Morsel) tuples for compatibility with SimpleCookie.update()
"""
if not header:
return []
cookies: list[tuple[str, Morsel[str]]] = []
morsel: Morsel[str]
i = 0
n = len(header)
invalid_names = []
while i < n:
# Use the same pattern as parse_set_cookie_headers to find cookies
match = _COOKIE_PATTERN.match(header, i)
if not match:
# Fallback for malformed cookies https://github.com/aio-libs/aiohttp/issues/11632
# Find next semicolon to skip or attempt simple key=value parsing
next_semi = header.find(";", i)
eq_pos = header.find("=", i)
# Try to extract key=value if '=' comes before ';'
if eq_pos != -1 and (next_semi == -1 or eq_pos < next_semi):
end_pos = next_semi if next_semi != -1 else n
key = header[i:eq_pos].strip()
value = header[eq_pos + 1 : end_pos].strip()
# Validate the name (same as regex path)
if not _COOKIE_NAME_RE.match(key):
invalid_names.append(key)
else:
morsel = Morsel()
try:
morsel.__setstate__( # type: ignore[attr-defined]
{
"key": key,
"value": _unquote(value),
"coded_value": value,
}
)
except CookieError:
pass
else:
cookies.append((key, morsel))
# Move to next cookie or end
i = next_semi + 1 if next_semi != -1 else n
continue
key = match.group("key")
value = match.group("val") or ""
i = match.end(0)
# Validate the name
if not key or not _COOKIE_NAME_RE.match(key):
invalid_names.append(key)
continue
# Create new morsel
morsel = Morsel()
# Preserve the original value as coded_value (with quotes if present)
# We use __setstate__ instead of the public set() API because it allows us to
# bypass validation and set already validated state. This is more stable than
# setting protected attributes directly and unlikely to change since it would
# break pickling.
try:
morsel.__setstate__( # type: ignore[attr-defined]
{"key": key, "value": _unquote(value), "coded_value": value}
)
except CookieError:
continue
cookies.append((key, morsel))
if invalid_names:
internal_logger.debug(
"Cannot load cookie. Illegal cookie names: %r", invalid_names
)
return cookies
def parse_set_cookie_headers(headers: Sequence[str]) -> list[tuple[str, Morsel[str]]]:
"""
Parse cookie headers using a vendored version of SimpleCookie parsing.
This implementation is based on SimpleCookie.__parse_string to ensure
compatibility with how SimpleCookie parses cookies, including handling
of malformed cookies with missing semicolons.
This function is used for both Cookie and Set-Cookie headers in order to be
forgiving. Ideally we would have followed RFC 6265 Section 5.2 (for Cookie
headers) and RFC 6265 Section 4.2.1 (for Set-Cookie headers), but the
real world data makes it impossible since we need to be a bit more forgiving.
NOTE: This implementation differs from SimpleCookie in handling unmatched quotes.
SimpleCookie will stop parsing when it encounters a cookie value with an unmatched
quote (e.g., 'cookie="value'), causing subsequent cookies to be silently dropped.
This implementation handles unmatched quotes more gracefully to prevent cookie loss.
See https://github.com/aio-libs/aiohttp/issues/7993
"""
parsed_cookies: list[tuple[str, Morsel[str]]] = []
for header in headers:
if not header:
continue
# Parse cookie string using SimpleCookie's algorithm
i = 0
n = len(header)
current_morsel: Morsel[str] | None = None
morsel_seen = False
while 0 <= i < n:
# Start looking for a cookie
match = _COOKIE_PATTERN.match(header, i)
if not match:
# No more cookies
break
key, value = match.group("key"), match.group("val")
i = match.end(0)
lower_key = key.lower()
if key[0] == "$":
if not morsel_seen:
# We ignore attributes which pertain to the cookie
# mechanism as a whole, such as "$Version".
continue
# Process as attribute
if current_morsel is not None:
attr_lower_key = lower_key[1:]
if attr_lower_key in _COOKIE_KNOWN_ATTRS:
current_morsel[attr_lower_key] = value or ""
elif lower_key in _COOKIE_KNOWN_ATTRS:
if not morsel_seen:
# Invalid cookie string - attribute before cookie
break
if lower_key in _COOKIE_BOOL_ATTRS:
# Boolean attribute with any value should be True
if current_morsel is not None and current_morsel.isReservedKey(key):
current_morsel[lower_key] = True
elif value is None:
# Invalid cookie string - non-boolean attribute without value
break
elif current_morsel is not None:
# Regular attribute with value
current_morsel[lower_key] = _unquote(value)
elif value is not None:
# This is a cookie name=value pair
# Validate the name
if key in _COOKIE_KNOWN_ATTRS or not _COOKIE_NAME_RE.match(key):
internal_logger.warning(
"Can not load cookies: Illegal cookie name %r", key
)
current_morsel = None
else:
# Create new morsel
current_morsel = Morsel()
# Preserve the original value as coded_value (with quotes if present)
try:
current_morsel.__setstate__( # type: ignore[attr-defined]
{
"key": key,
"value": _unquote(value),
"coded_value": value,
}
)
except CookieError:
current_morsel = None
else:
parsed_cookies.append((key, current_morsel))
morsel_seen = True
else:
# Invalid cookie string - no value for non-attribute
break
return parsed_cookies
+89
View File
@@ -0,0 +1,89 @@
# This file contains just the definitions needed in our Cython code.
from libc.stdint cimport uint8_t, uint16_t, uint64_t
cdef extern from "llhttp.h":
struct llhttp__internal_s:
void* data
uint64_t content_length
uint8_t type
uint8_t method
uint8_t http_major
uint8_t http_minor
uint8_t upgrade
uint16_t flags
uint16_t status_code
ctypedef llhttp__internal_s llhttp__internal_t
ctypedef llhttp__internal_t llhttp_t
ctypedef int (*llhttp_data_cb)(llhttp_t*, const char *at, size_t length) except -1
ctypedef int (*llhttp_cb)(llhttp_t*) except -1
struct llhttp_settings_s:
llhttp_cb on_message_begin
llhttp_data_cb on_url
llhttp_data_cb on_status
llhttp_data_cb on_header_field
llhttp_data_cb on_header_value
llhttp_cb on_headers_complete
llhttp_data_cb on_body
llhttp_cb on_message_complete
llhttp_cb on_chunk_header
llhttp_cb on_chunk_complete
ctypedef llhttp_settings_s llhttp_settings_t
enum llhttp_errno:
HPE_OK,
HPE_INVALID_METHOD,
HPE_INVALID_URL,
HPE_INVALID_CONSTANT,
HPE_INVALID_VERSION,
HPE_INVALID_HEADER_TOKEN,
HPE_INVALID_CONTENT_LENGTH,
HPE_INVALID_CHUNK_SIZE,
HPE_INVALID_STATUS,
HPE_INVALID_EOF_STATE,
HPE_INVALID_TRANSFER_ENCODING,
HPE_CB_MESSAGE_BEGIN,
HPE_CB_HEADERS_COMPLETE,
HPE_CB_MESSAGE_COMPLETE,
HPE_CB_CHUNK_HEADER,
HPE_CB_CHUNK_COMPLETE,
HPE_PAUSED,
HPE_PAUSED_UPGRADE
ctypedef llhttp_errno llhttp_errno_t
enum llhttp_flags:
F_CHUNKED,
F_CONTENT_LENGTH
enum llhttp_type:
HTTP_REQUEST,
HTTP_RESPONSE
enum llhttp_method:
HTTP_CONNECT
void llhttp_settings_init(llhttp_settings_t* settings)
void llhttp_init(llhttp_t* parser, llhttp_type type,
const llhttp_settings_t* settings)
llhttp_errno_t llhttp_execute(llhttp_t* parser, const char* data, size_t len)
int llhttp_should_keep_alive(const llhttp_t* parser)
void llhttp_resume(llhttp_t* parser)
void llhttp_resume_after_upgrade(llhttp_t* parser)
llhttp_errno_t llhttp_get_errno(const llhttp_t* parser)
const char* llhttp_get_error_reason(const llhttp_t* parser)
const char* llhttp_get_error_pos(const llhttp_t* parser)
void llhttp_set_lenient_headers(llhttp_t* parser, int enabled)
void llhttp_set_lenient_optional_cr_before_lf(llhttp_t* parser, int enabled)
void llhttp_set_lenient_spaces_after_chunk_size(llhttp_t* parser, int enabled)
+2
View File
@@ -0,0 +1,2 @@
cdef extern from "_find_header.h":
int find_header(char *, int)
+83
View File
@@ -0,0 +1,83 @@
# The file is autogenerated from aiohttp/hdrs.py
# Run ./tools/gen.py to update it after the origin changing.
from . import hdrs
cdef tuple headers = (
hdrs.ACCEPT,
hdrs.ACCEPT_CHARSET,
hdrs.ACCEPT_ENCODING,
hdrs.ACCEPT_LANGUAGE,
hdrs.ACCEPT_RANGES,
hdrs.ACCESS_CONTROL_ALLOW_CREDENTIALS,
hdrs.ACCESS_CONTROL_ALLOW_HEADERS,
hdrs.ACCESS_CONTROL_ALLOW_METHODS,
hdrs.ACCESS_CONTROL_ALLOW_ORIGIN,
hdrs.ACCESS_CONTROL_EXPOSE_HEADERS,
hdrs.ACCESS_CONTROL_MAX_AGE,
hdrs.ACCESS_CONTROL_REQUEST_HEADERS,
hdrs.ACCESS_CONTROL_REQUEST_METHOD,
hdrs.AGE,
hdrs.ALLOW,
hdrs.AUTHORIZATION,
hdrs.CACHE_CONTROL,
hdrs.CONNECTION,
hdrs.CONTENT_DISPOSITION,
hdrs.CONTENT_ENCODING,
hdrs.CONTENT_LANGUAGE,
hdrs.CONTENT_LENGTH,
hdrs.CONTENT_LOCATION,
hdrs.CONTENT_MD5,
hdrs.CONTENT_RANGE,
hdrs.CONTENT_TRANSFER_ENCODING,
hdrs.CONTENT_TYPE,
hdrs.COOKIE,
hdrs.DATE,
hdrs.DESTINATION,
hdrs.DIGEST,
hdrs.ETAG,
hdrs.EXPECT,
hdrs.EXPIRES,
hdrs.FORWARDED,
hdrs.FROM,
hdrs.HOST,
hdrs.IF_MATCH,
hdrs.IF_MODIFIED_SINCE,
hdrs.IF_NONE_MATCH,
hdrs.IF_RANGE,
hdrs.IF_UNMODIFIED_SINCE,
hdrs.KEEP_ALIVE,
hdrs.LAST_EVENT_ID,
hdrs.LAST_MODIFIED,
hdrs.LINK,
hdrs.LOCATION,
hdrs.MAX_FORWARDS,
hdrs.ORIGIN,
hdrs.PRAGMA,
hdrs.PROXY_AUTHENTICATE,
hdrs.PROXY_AUTHORIZATION,
hdrs.RANGE,
hdrs.REFERER,
hdrs.RETRY_AFTER,
hdrs.SEC_WEBSOCKET_ACCEPT,
hdrs.SEC_WEBSOCKET_EXTENSIONS,
hdrs.SEC_WEBSOCKET_KEY,
hdrs.SEC_WEBSOCKET_KEY1,
hdrs.SEC_WEBSOCKET_PROTOCOL,
hdrs.SEC_WEBSOCKET_VERSION,
hdrs.SERVER,
hdrs.SET_COOKIE,
hdrs.TE,
hdrs.TRAILER,
hdrs.TRANSFER_ENCODING,
hdrs.URI,
hdrs.UPGRADE,
hdrs.USER_AGENT,
hdrs.VARY,
hdrs.VIA,
hdrs.WWW_AUTHENTICATE,
hdrs.WANT_DIGEST,
hdrs.WARNING,
hdrs.X_FORWARDED_FOR,
hdrs.X_FORWARDED_HOST,
hdrs.X_FORWARDED_PROTO,
)
Binary file not shown.
+984
View File
@@ -0,0 +1,984 @@
# Based on https://github.com/MagicStack/httptools
#
from cpython cimport (
Py_buffer,
PyBUF_SIMPLE,
PyBuffer_Release,
PyBytes_AsString,
PyBytes_AsStringAndSize,
PyObject_GetBuffer,
)
from cpython.mem cimport PyMem_Free, PyMem_Malloc
from libc.limits cimport ULLONG_MAX
from libc.string cimport memcpy
from multidict import CIMultiDict as _CIMultiDict, CIMultiDictProxy as _CIMultiDictProxy
from yarl import URL as _URL
from aiohttp import hdrs
from aiohttp.helpers import DEBUG, set_exception
from .http_exceptions import (
BadHttpMessage,
BadHttpMethod,
BadStatusLine,
ContentLengthError,
InvalidHeader,
InvalidURLError,
LineTooLong,
PayloadEncodingError,
TransferEncodingError,
)
from .http_parser import DeflateBuffer as _DeflateBuffer
from .http_writer import (
HttpVersion as _HttpVersion,
HttpVersion10 as _HttpVersion10,
HttpVersion11 as _HttpVersion11,
)
from .streams import EMPTY_PAYLOAD as _EMPTY_PAYLOAD, StreamReader as _StreamReader
cimport cython
from aiohttp cimport _cparser as cparser
include "_headers.pxi"
from aiohttp cimport _find_header
cdef frozenset ALLOWED_UPGRADES = frozenset({"websocket"})
DEF DEFAULT_FREELIST_SIZE = 250
cdef extern from "Python.h":
int PyByteArray_Resize(object, Py_ssize_t) except -1
Py_ssize_t PyByteArray_Size(object) except -1
char* PyByteArray_AsString(object)
__all__ = ('HttpRequestParser', 'HttpResponseParser',
'RawRequestMessage', 'RawResponseMessage')
cdef object URL = _URL
cdef object URL_build = URL.build
cdef object CIMultiDict = _CIMultiDict
cdef object CIMultiDictProxy = _CIMultiDictProxy
cdef object HttpVersion = _HttpVersion
cdef object HttpVersion10 = _HttpVersion10
cdef object HttpVersion11 = _HttpVersion11
cdef object SEC_WEBSOCKET_KEY1 = hdrs.SEC_WEBSOCKET_KEY1
cdef object CONTENT_ENCODING = hdrs.CONTENT_ENCODING
cdef object EMPTY_PAYLOAD = _EMPTY_PAYLOAD
cdef object StreamReader = _StreamReader
cdef object DeflateBuffer = _DeflateBuffer
cdef tuple EMPTY_FEED_DATA_RESULT = ((), False, b"")
# RFC 9110 singleton headers — duplicates are rejected in strict mode.
# In lax mode (response parser default), the check is skipped entirely
# since real-world servers (e.g. Google APIs, Werkzeug) commonly send
# duplicate headers like Content-Type or Server.
cdef frozenset SINGLETON_HEADERS = frozenset({
hdrs.CONTENT_LENGTH,
hdrs.CONTENT_LOCATION,
hdrs.CONTENT_RANGE,
hdrs.CONTENT_TYPE,
hdrs.ETAG,
hdrs.HOST,
hdrs.MAX_FORWARDS,
hdrs.SERVER,
hdrs.TRANSFER_ENCODING,
hdrs.USER_AGENT,
})
cdef inline object extend(object buf, const char* at, size_t length):
cdef Py_ssize_t s
cdef char* ptr
s = PyByteArray_Size(buf)
PyByteArray_Resize(buf, s + length)
ptr = PyByteArray_AsString(buf)
memcpy(ptr + s, at, length)
# The method-name table and its length come straight from llhttp's canonical
# HTTP_ALL_METHOD_MAP, so they track the vendored llhttp version automatically
# instead of relying on a hand-maintained method count.
cdef extern from *:
"""
#include "llhttp.h"
#define _AIOHTTP_METHOD_NAME(NUM, NAME, STRING) [NUM] = #STRING,
static const char* const _aiohttp_method_names[] = {
HTTP_ALL_METHOD_MAP(_AIOHTTP_METHOD_NAME)
};
#undef _AIOHTTP_METHOD_NAME
"""
const char* _aiohttp_method_names[]
const int METHODS_COUNT "((int)(sizeof(_aiohttp_method_names) / sizeof(_aiohttp_method_names[0])))"
cdef list _http_method = []
for i in range(METHODS_COUNT):
assert _aiohttp_method_names[i] is not NULL
_http_method.append(_aiohttp_method_names[i].decode('ascii'))
cdef inline object find_header(bytes raw_header):
cdef Py_ssize_t size
cdef char *buf
cdef int idx
PyBytes_AsStringAndSize(raw_header, &buf, &size)
idx = _find_header.find_header(buf, size)
if idx == -1:
return raw_header.decode('utf-8', 'surrogateescape')
return headers[idx]
@cython.freelist(DEFAULT_FREELIST_SIZE)
cdef class RawRequestMessage:
cdef readonly str method
cdef readonly str path
cdef readonly object version # HttpVersion
cdef readonly object headers # CIMultiDict
cdef readonly object raw_headers # tuple
cdef readonly object should_close
cdef readonly object compression
cdef readonly object upgrade
cdef readonly object chunked
cdef readonly object url # yarl.URL
def __init__(self, method, path, version, headers, raw_headers,
should_close, compression, upgrade, chunked, url):
self.method = method
self.path = path
self.version = version
self.headers = headers
self.raw_headers = raw_headers
self.should_close = should_close
self.compression = compression
self.upgrade = upgrade
self.chunked = chunked
self.url = url
def __repr__(self):
info = []
info.append(("method", self.method))
info.append(("path", self.path))
info.append(("version", self.version))
info.append(("headers", self.headers))
info.append(("raw_headers", self.raw_headers))
info.append(("should_close", self.should_close))
info.append(("compression", self.compression))
info.append(("upgrade", self.upgrade))
info.append(("chunked", self.chunked))
info.append(("url", self.url))
sinfo = ', '.join(name + '=' + repr(val) for name, val in info)
return '<RawRequestMessage(' + sinfo + ')>'
def _replace(self, **dct):
cdef RawRequestMessage ret
ret = _new_request_message(self.method,
self.path,
self.version,
self.headers,
self.raw_headers,
self.should_close,
self.compression,
self.upgrade,
self.chunked,
self.url)
if "method" in dct:
ret.method = dct["method"]
if "path" in dct:
ret.path = dct["path"]
if "version" in dct:
ret.version = dct["version"]
if "headers" in dct:
ret.headers = dct["headers"]
if "raw_headers" in dct:
ret.raw_headers = dct["raw_headers"]
if "should_close" in dct:
ret.should_close = dct["should_close"]
if "compression" in dct:
ret.compression = dct["compression"]
if "upgrade" in dct:
ret.upgrade = dct["upgrade"]
if "chunked" in dct:
ret.chunked = dct["chunked"]
if "url" in dct:
ret.url = dct["url"]
return ret
cdef _new_request_message(str method,
str path,
object version,
object headers,
object raw_headers,
bint should_close,
object compression,
bint upgrade,
bint chunked,
object url):
cdef RawRequestMessage ret
ret = RawRequestMessage.__new__(RawRequestMessage)
ret.method = method
ret.path = path
ret.version = version
ret.headers = headers
ret.raw_headers = raw_headers
ret.should_close = should_close
ret.compression = compression
ret.upgrade = upgrade
ret.chunked = chunked
ret.url = url
return ret
@cython.freelist(DEFAULT_FREELIST_SIZE)
cdef class RawResponseMessage:
cdef readonly object version # HttpVersion
cdef readonly int code
cdef readonly str reason
cdef readonly object headers # CIMultiDict
cdef readonly object raw_headers # tuple
cdef readonly object should_close
cdef readonly object compression
cdef readonly object upgrade
cdef readonly object chunked
def __init__(self, version, code, reason, headers, raw_headers,
should_close, compression, upgrade, chunked):
self.version = version
self.code = code
self.reason = reason
self.headers = headers
self.raw_headers = raw_headers
self.should_close = should_close
self.compression = compression
self.upgrade = upgrade
self.chunked = chunked
def __repr__(self):
info = []
info.append(("version", self.version))
info.append(("code", self.code))
info.append(("reason", self.reason))
info.append(("headers", self.headers))
info.append(("raw_headers", self.raw_headers))
info.append(("should_close", self.should_close))
info.append(("compression", self.compression))
info.append(("upgrade", self.upgrade))
info.append(("chunked", self.chunked))
sinfo = ', '.join(name + '=' + repr(val) for name, val in info)
return '<RawResponseMessage(' + sinfo + ')>'
cdef _new_response_message(object version,
int code,
str reason,
object headers,
object raw_headers,
bint should_close,
object compression,
bint upgrade,
bint chunked):
cdef RawResponseMessage ret
ret = RawResponseMessage.__new__(RawResponseMessage)
ret.version = version
ret.code = code
ret.reason = reason
ret.headers = headers
ret.raw_headers = raw_headers
ret.should_close = should_close
ret.compression = compression
ret.upgrade = upgrade
ret.chunked = chunked
return ret
@cython.internal
cdef class HttpParser:
cdef:
cparser.llhttp_t* _cparser
cparser.llhttp_settings_t* _csettings
bytes _raw_name
object _name
bytes _raw_value
bint _has_value
int _header_name_size
readonly object protocol
object _loop
object _timer
size_t _max_line_size
size_t _max_field_size
size_t _max_headers
bint _response_with_body
bint _read_until_eof
bint _lax
bytes _tail
bint _started
object _url
bytearray _buf
str _path
str _reason
list _headers
set _seen_singletons
list _raw_headers
bint _upgraded
bint _pending_upgrade
list _messages
bint _more_data_available
bint _paused
Py_ssize_t _msg_in_flight
Py_ssize_t _max_msg_queue_size
bint _eof_pending
object _payload
unsigned long long _content_length_expected
bint _payload_error
object _payload_exception
object _last_error
bint _auto_decompress
int _limit
str _content_encoding
Py_buffer py_buf
def __cinit__(self):
self._cparser = <cparser.llhttp_t*> \
PyMem_Malloc(sizeof(cparser.llhttp_t))
if self._cparser is NULL:
raise MemoryError()
self._csettings = <cparser.llhttp_settings_t*> \
PyMem_Malloc(sizeof(cparser.llhttp_settings_t))
if self._csettings is NULL:
raise MemoryError()
def __dealloc__(self):
PyMem_Free(self._cparser)
PyMem_Free(self._csettings)
cdef _init(
self, cparser.llhttp_type mode,
object protocol, object loop, int limit,
object timer=None,
size_t max_line_size=8190, size_t max_headers=128,
size_t max_field_size=8190, payload_exception=None,
bint response_with_body=True, bint read_until_eof=False,
bint auto_decompress=True,
Py_ssize_t max_msg_queue_size=0,
):
cparser.llhttp_settings_init(self._csettings)
cparser.llhttp_init(self._cparser, mode, self._csettings)
self._cparser.data = <void*>self
self._cparser.content_length = 0
self._content_length_expected = 0
self.protocol = protocol
self._loop = loop
self._timer = timer
self._buf = bytearray()
self._more_data_available = False
self._paused = False
self._msg_in_flight = 0
self._max_msg_queue_size = max_msg_queue_size
self._eof_pending = False
self._payload = None
self._payload_error = 0
self._payload_exception = payload_exception
self._messages = []
self._raw_name = b""
self._raw_value = b""
self._tail = b""
self._has_value = False
self._header_name_size = 0
self._max_line_size = max_line_size
self._max_headers = max_headers
self._max_field_size = max_field_size
self._response_with_body = response_with_body
self._read_until_eof = read_until_eof
self._upgraded = False
self._pending_upgrade = False
self._auto_decompress = auto_decompress
self._content_encoding = None
self._lax = False
self._seen_singletons = set()
self._csettings.on_url = cb_on_url
self._csettings.on_status = cb_on_status
self._csettings.on_header_field = cb_on_header_field
self._csettings.on_header_value = cb_on_header_value
self._csettings.on_headers_complete = cb_on_headers_complete
self._csettings.on_body = cb_on_body
self._csettings.on_message_begin = cb_on_message_begin
self._csettings.on_message_complete = cb_on_message_complete
self._csettings.on_chunk_header = cb_on_chunk_header
self._csettings.on_chunk_complete = cb_on_chunk_complete
self._last_error = None
self._limit = limit
cdef _process_header(self):
cdef str value
if self._raw_name != b"":
name = find_header(self._raw_name)
value = self._raw_value.decode('utf-8', 'surrogateescape')
# reject null bytes in header values - matches the Python parser
# check at http_parser.py. llhttp in lenient mode doesn't reject
# these itself, so we need to catch them here.
# ref: RFC 9110 section 5.5 (CTL chars forbidden in field values)
if "\x00" in value:
raise InvalidHeader(self._raw_value)
if not self._lax and name in SINGLETON_HEADERS:
if name in self._seen_singletons:
raise BadHttpMessage(f"Duplicate '{name}' header found.")
self._seen_singletons.add(name)
self._headers.append((name, value))
if len(self._headers) > self._max_headers:
raise BadHttpMessage("Too many headers received")
if name is CONTENT_ENCODING:
self._content_encoding = value
self._has_value = False
self._header_name_size = 0
self._raw_headers.append((self._raw_name, self._raw_value))
self._raw_name = b""
self._raw_value = b""
cdef _on_header_field(self, char* at, size_t length):
if self._has_value:
self._process_header()
if self._raw_name == b"":
self._raw_name = at[:length]
else:
self._raw_name += at[:length]
cdef _on_header_value(self, char* at, size_t length):
if self._raw_value == b"":
self._raw_value = at[:length]
else:
self._raw_value += at[:length]
self._has_value = True
cdef _on_headers_complete(self):
cdef str h_upg
cdef str enc
self._process_header()
http_version = self.http_version()
should_close = not cparser.llhttp_should_keep_alive(self._cparser)
upgrade = self._cparser.upgrade
chunked = self._cparser.flags & cparser.F_CHUNKED
raw_headers = tuple(self._raw_headers)
headers = CIMultiDictProxy(CIMultiDict(self._headers))
if self._cparser.type == cparser.HTTP_REQUEST:
if http_version == HttpVersion11 and hdrs.HOST not in headers:
raise BadHttpMessage("Missing 'Host' header in request.")
h_upg = headers.get("upgrade", "")
if (upgrade and h_upg.isascii() and h_upg.lower() in ALLOWED_UPGRADES) or self._cparser.method == cparser.HTTP_CONNECT:
# https://www.rfc-editor.org/info/rfc9110/#section-7.8-15
# Defer the protocol switch until the complete request has been
# received.
self._pending_upgrade = True
else:
if upgrade and self._cparser.status_code == 101:
# llhttp pauses for a 101 on its own; just mark the pending
# switch so feed_data returns the upgraded-protocol tail.
self._pending_upgrade = True
# do not support old websocket spec
if SEC_WEBSOCKET_KEY1 in headers:
raise InvalidHeader(SEC_WEBSOCKET_KEY1)
encoding = None
enc = self._content_encoding
if enc is not None:
self._content_encoding = None
if enc.isascii() and enc.lower() in {"gzip", "deflate", "br", "zstd"}:
encoding = enc
if self._cparser.type == cparser.HTTP_REQUEST:
method = <str>_http_method[self._cparser.method]
msg = _new_request_message(
method, self._path,
http_version, headers, raw_headers,
should_close, encoding, upgrade, chunked, self._url)
else:
msg = _new_response_message(
http_version, self._cparser.status_code, self._reason,
headers, raw_headers, should_close, encoding,
upgrade, chunked)
if (
self._response_with_body
and (
ULLONG_MAX > self._cparser.content_length > 0 or chunked or
self._cparser.method == cparser.HTTP_CONNECT or
(self._cparser.status_code >= 199 and
self._cparser.content_length == 0 and
self._read_until_eof)
)
):
payload = StreamReader(
self.protocol, timer=self._timer, loop=self._loop,
limit=self._limit)
else:
payload = EMPTY_PAYLOAD
self._payload = payload
self._content_length_expected = self._cparser.content_length
if encoding is not None and self._auto_decompress:
self._payload = DeflateBuffer(payload, encoding, max_decompress_size=self._limit)
self._messages.append((msg, payload))
cdef _on_message_complete(self):
self._payload.feed_eof()
self._payload = None
cdef _on_chunk_header(self):
self._payload.begin_http_chunk_receiving()
cdef _on_chunk_complete(self):
self._payload.end_http_chunk_receiving()
cdef object _on_status_complete(self):
pass
cdef inline http_version(self):
cdef cparser.llhttp_t* parser = self._cparser
if parser.http_major == 1:
if parser.http_minor == 0:
return HttpVersion10
elif parser.http_minor == 1:
return HttpVersion11
return HttpVersion(parser.http_major, parser.http_minor)
### Public API ###
def pause_reading(self):
assert self._payload is not None
self._paused = True
def message_consumed(self):
# Protocol drained a queued message; free a slot for parsing.
if self._msg_in_flight > 0:
self._msg_in_flight -= 1
def feed_eof(self):
cdef bytes desc
if self._payload is not None:
if self._cparser.flags & cparser.F_CHUNKED:
raise TransferEncodingError(
"Not enough data to satisfy transfer length header.")
elif self._cparser.flags & cparser.F_CONTENT_LENGTH:
received = self._content_length_expected - self._cparser.content_length
raise ContentLengthError(
f"Not enough data to satisfy content length header "
f"(received {received} of {self._content_length_expected} bytes).")
elif cparser.llhttp_get_errno(self._cparser) != cparser.HPE_OK:
desc = cparser.llhttp_get_error_reason(self._cparser)
raise PayloadEncodingError(desc.decode('latin-1'))
else:
self._eof_pending = True
while self._more_data_available:
if self._paused:
self._paused = False
return # Will resume via feed_data(b"") later
self._more_data_available = self._payload.feed_data(b"", 0)
self._payload.feed_eof()
self._payload = None
self._more_data_available = False
self._eof_pending = False
elif self._started:
self._on_headers_complete()
if self._messages:
return self._messages[-1][0]
def feed_data(self, incoming_data):
cdef:
size_t data_len
size_t nb
char* base
cdef cparser.llhttp_errno_t errno
cdef bytes data
# Proactor loop sends bytearray.
# Ensure cython sees `data` as bytes
if type(incoming_data) is not bytes:
data = bytes(incoming_data)
else:
data = incoming_data
if self._tail:
data, self._tail = self._tail + data, b""
if self._more_data_available:
result = cb_on_body(self._cparser, b"", 0)
if result is cparser.HPE_PAUSED:
self._tail = data
return EMPTY_FEED_DATA_RESULT
if self._eof_pending:
self._payload.feed_eof()
self._payload = None
self._eof_pending = False
# We can't have new messages here, otherwise we wouldn't have
# received EOF.
return EMPTY_FEED_DATA_RESULT
PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE)
# Cache buffer pointer before PyBuffer_Release to avoid use-after-release.
base = <char*>self.py_buf.buf
data_len = <size_t>self.py_buf.len
errno = cparser.llhttp_execute(
self._cparser,
base,
data_len)
if errno is cparser.HPE_PAUSED_UPGRADE:
cparser.llhttp_resume_after_upgrade(self._cparser)
nb = cparser.llhttp_get_error_pos(self._cparser) - base
if self._pending_upgrade:
# A supported upgrade whose request body has now been fully read.
self._upgraded = True
self._pending_upgrade = False
elif errno is cparser.HPE_PAUSED:
cparser.llhttp_resume(self._cparser)
pos = cparser.llhttp_get_error_pos(self._cparser) - base
self._tail = data[pos:]
PyBuffer_Release(&self.py_buf)
if errno not in (cparser.HPE_OK, cparser.HPE_PAUSED, cparser.HPE_PAUSED_UPGRADE):
if self._payload_error == 0:
if self._last_error is not None:
ex = self._last_error
self._last_error = None
else:
error_pos = cparser.llhttp_get_error_pos(self._cparser)
error_off = error_pos - base
before = data[:error_off]
after = data[error_off:].split(b"\r\n", 1)[0]
before = before.rsplit(b"\r\n", 1)[-1]
data = before + after
pointer = " " * (len(repr(before))-1) + "^"
ex = parser_error_from_errno(self._cparser, data, pointer)
self._payload = None
raise ex
if self._messages:
messages = self._messages
self._messages = []
else:
messages = ()
if self._upgraded:
return messages, True, data[nb:]
if not messages: # Shortcut to reduce Python overhead
return EMPTY_FEED_DATA_RESULT
return messages, False, b""
def set_upgraded(self, val):
self._upgraded = val
cdef class HttpRequestParser(HttpParser):
def __init__(
self, protocol, loop, int limit, timer=None,
size_t max_line_size=8190, size_t max_headers=128,
size_t max_field_size=8190, payload_exception=None,
bint response_with_body=True, bint read_until_eof=False,
bint auto_decompress=True, Py_ssize_t max_msg_queue_size=0,
):
self._init(cparser.HTTP_REQUEST, protocol, loop, limit, timer,
max_line_size, max_headers, max_field_size,
payload_exception, response_with_body, read_until_eof,
auto_decompress, max_msg_queue_size)
cdef object _on_status_complete(self):
cdef int idx1, idx2
if not self._buf:
return
self._path = self._buf.decode('utf-8', 'surrogateescape')
try:
idx3 = len(self._path)
if self._cparser.method == cparser.HTTP_CONNECT:
# authority-form,
# https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3
self._url = URL.build(authority=self._path, encoded=True)
elif idx3 > 1 and self._path[0] == '/':
# origin-form,
# https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.1
idx1 = self._path.find("?")
if idx1 == -1:
query = ""
idx2 = self._path.find("#")
if idx2 == -1:
path = self._path
fragment = ""
else:
path = self._path[0: idx2]
fragment = self._path[idx2+1:]
else:
path = self._path[0:idx1]
idx1 += 1
idx2 = self._path.find("#", idx1)
if idx2 == -1:
query = self._path[idx1:]
fragment = ""
else:
query = self._path[idx1: idx2]
fragment = self._path[idx2+1:]
self._url = URL.build(
path=path,
query_string=query,
fragment=fragment,
encoded=True,
)
else:
# absolute-form for proxy maybe,
# https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2
self._url = URL(self._path, encoded=True)
finally:
PyByteArray_Resize(self._buf, 0)
cdef class HttpResponseParser(HttpParser):
def __init__(
self, protocol, loop, int limit, timer=None,
size_t max_line_size=8190, size_t max_headers=128,
size_t max_field_size=8190, payload_exception=None,
bint response_with_body=True, bint read_until_eof=False,
bint auto_decompress=True
):
self._init(cparser.HTTP_RESPONSE, protocol, loop, limit, timer,
max_line_size, max_headers, max_field_size,
payload_exception, response_with_body, read_until_eof,
auto_decompress)
# Use strict parsing on dev mode, so users are warned about broken servers.
if not DEBUG:
cparser.llhttp_set_lenient_headers(self._cparser, 1)
cparser.llhttp_set_lenient_optional_cr_before_lf(self._cparser, 1)
cparser.llhttp_set_lenient_spaces_after_chunk_size(self._cparser, 1)
self._lax = True
cdef object _on_status_complete(self):
if self._buf:
self._reason = self._buf.decode('utf-8', 'surrogateescape')
PyByteArray_Resize(self._buf, 0)
else:
self._reason = self._reason or ''
cdef int cb_on_message_begin(cparser.llhttp_t* parser) except -1:
cdef HttpParser pyparser = <HttpParser>parser.data
pyparser._started = True
pyparser._headers = []
pyparser._seen_singletons = set()
pyparser._raw_headers = []
PyByteArray_Resize(pyparser._buf, 0)
pyparser._path = None
pyparser._reason = None
return 0
cdef int cb_on_url(cparser.llhttp_t* parser,
const char *at, size_t length) except -1:
cdef HttpParser pyparser = <HttpParser>parser.data
try:
if len(pyparser._buf) + length > pyparser._max_line_size:
status = pyparser._buf + at[:length]
raise LineTooLong(status[:100] + b"...", pyparser._max_line_size)
extend(pyparser._buf, at, length)
except BaseException as ex:
pyparser._last_error = ex
return -1
else:
return 0
cdef int cb_on_status(cparser.llhttp_t* parser,
const char *at, size_t length) except -1:
cdef HttpParser pyparser = <HttpParser>parser.data
try:
if len(pyparser._buf) + length > pyparser._max_line_size:
reason = pyparser._buf + at[:length]
raise LineTooLong(reason[:100] + b"...", pyparser._max_line_size)
extend(pyparser._buf, at, length)
except BaseException as ex:
pyparser._last_error = ex
return -1
else:
return 0
cdef int cb_on_header_field(cparser.llhttp_t* parser,
const char *at, size_t length) except -1:
cdef HttpParser pyparser = <HttpParser>parser.data
cdef Py_ssize_t size
try:
pyparser._on_status_complete()
size = len(pyparser._raw_name) + length
if size > pyparser._max_field_size:
name = pyparser._raw_name + at[:length]
raise LineTooLong(name[:100] + b"...", pyparser._max_field_size)
pyparser._header_name_size = size
pyparser._on_header_field(at, length)
except BaseException as ex:
pyparser._last_error = ex
return -1
else:
return 0
cdef int cb_on_header_value(cparser.llhttp_t* parser,
const char *at, size_t length) except -1:
cdef HttpParser pyparser = <HttpParser>parser.data
cdef Py_ssize_t size
try:
size = len(pyparser._raw_value) + length
if pyparser._header_name_size + size > pyparser._max_field_size:
value = pyparser._raw_value + at[:length]
raise LineTooLong(value[:100] + b"...", pyparser._max_field_size)
pyparser._on_header_value(at, length)
except BaseException as ex:
pyparser._last_error = ex
return -1
else:
return 0
cdef int cb_on_headers_complete(cparser.llhttp_t* parser) except -1:
cdef HttpParser pyparser = <HttpParser>parser.data
try:
pyparser._on_status_complete()
pyparser._on_headers_complete()
except BaseException as exc:
pyparser._last_error = exc
return -1
else:
if not pyparser._response_with_body:
return 1
return 0
cdef int cb_on_body(cparser.llhttp_t* parser,
const char *at, size_t length) except -1:
cdef HttpParser pyparser = <HttpParser>parser.data
cdef bytes body = at[:length]
while body or pyparser._more_data_available:
try:
pyparser._more_data_available = pyparser._payload.feed_data(body, length)
except BaseException as underlying_exc:
reraised_exc = underlying_exc
if pyparser._payload_exception is not None:
reraised_exc = pyparser._payload_exception(str(underlying_exc))
set_exception(pyparser._payload, reraised_exc, underlying_exc)
pyparser._payload_error = 1
pyparser._paused = False
return -1
body = b""
length = 0
if pyparser._paused:
pyparser._paused = False
return cparser.HPE_PAUSED
pyparser._paused = False
return 0
cdef int cb_on_message_complete(cparser.llhttp_t* parser) except -1:
cdef HttpParser pyparser = <HttpParser>parser.data
try:
pyparser._started = False
pyparser._on_message_complete()
except BaseException as exc:
pyparser._last_error = exc
return -1
else:
if pyparser._max_msg_queue_size:
pyparser._msg_in_flight += 1
if pyparser._msg_in_flight >= pyparser._max_msg_queue_size:
# Queue full: pause llhttp between messages. feed_data() buffers
# the remainder as tail; resumes once the queue drains.
return cparser.HPE_PAUSED
return 0
cdef int cb_on_chunk_header(cparser.llhttp_t* parser) except -1:
cdef HttpParser pyparser = <HttpParser>parser.data
try:
pyparser._on_chunk_header()
except BaseException as exc:
pyparser._last_error = exc
return -1
else:
return 0
cdef int cb_on_chunk_complete(cparser.llhttp_t* parser) except -1:
cdef HttpParser pyparser = <HttpParser>parser.data
try:
pyparser._on_chunk_complete()
except BaseException as exc:
pyparser._last_error = exc
return -1
else:
return 0
cdef parser_error_from_errno(cparser.llhttp_t* parser, data, pointer):
cdef cparser.llhttp_errno_t errno = cparser.llhttp_get_errno(parser)
cdef bytes desc = cparser.llhttp_get_error_reason(parser)
err_msg = "{}:\n\n {!r}\n {}".format(desc.decode("latin-1"), data, pointer)
if errno in {cparser.HPE_CB_MESSAGE_BEGIN,
cparser.HPE_CB_HEADERS_COMPLETE,
cparser.HPE_CB_MESSAGE_COMPLETE,
cparser.HPE_CB_CHUNK_HEADER,
cparser.HPE_CB_CHUNK_COMPLETE,
cparser.HPE_INVALID_HEADER_TOKEN,
cparser.HPE_INVALID_CONTENT_LENGTH,
cparser.HPE_INVALID_CHUNK_SIZE,
cparser.HPE_INVALID_EOF_STATE,
cparser.HPE_INVALID_TRANSFER_ENCODING}:
return BadHttpMessage(err_msg)
elif errno == cparser.HPE_INVALID_METHOD:
if data.startswith(b"\x16\x03"):
return BadHttpMethod(error="Received HTTPS traffic on an HTTP port")
return BadHttpMethod(error=err_msg)
elif errno in {cparser.HPE_INVALID_STATUS,
cparser.HPE_INVALID_VERSION,
cparser.HPE_INVALID_CONSTANT}:
return BadStatusLine(error=f"Bad status line:\n {err_msg}")
elif errno == cparser.HPE_INVALID_URL:
return InvalidURLError(err_msg)
return BadHttpMessage(err_msg)
Binary file not shown.
+164
View File
@@ -0,0 +1,164 @@
from cpython.bytes cimport PyBytes_FromStringAndSize
from cpython.exc cimport PyErr_NoMemory
from cpython.mem cimport PyMem_Free, PyMem_Malloc, PyMem_Realloc
from cpython.object cimport PyObject_Str
from libc.stdint cimport uint8_t, uint64_t
from libc.string cimport memcpy
from multidict import istr
DEF BUF_SIZE = 16 * 1024 # 16KiB
cdef object _istr = istr
# ----------------- writer ---------------------------
cdef struct Writer:
char *buf
Py_ssize_t size
Py_ssize_t pos
bint heap_allocated
cdef inline void _init_writer(Writer* writer, char *buf):
writer.buf = buf
writer.size = BUF_SIZE
writer.pos = 0
writer.heap_allocated = 0
cdef inline void _release_writer(Writer* writer):
if writer.heap_allocated:
PyMem_Free(writer.buf)
cdef inline int _write_byte(Writer* writer, uint8_t ch):
cdef char * buf
cdef Py_ssize_t size
if writer.pos == writer.size:
# reallocate
size = writer.size + BUF_SIZE
if not writer.heap_allocated:
buf = <char*>PyMem_Malloc(size)
if buf == NULL:
PyErr_NoMemory()
return -1
memcpy(buf, writer.buf, writer.size)
else:
buf = <char*>PyMem_Realloc(writer.buf, size)
if buf == NULL:
PyErr_NoMemory()
return -1
writer.buf = buf
writer.size = size
writer.heap_allocated = 1
writer.buf[writer.pos] = <char>ch
writer.pos += 1
return 0
cdef inline int _write_utf8(Writer* writer, Py_UCS4 symbol):
cdef uint64_t utf = <uint64_t> symbol
if utf < 0x80:
return _write_byte(writer, <uint8_t>utf)
elif utf < 0x800:
if _write_byte(writer, <uint8_t>(0xc0 | (utf >> 6))) < 0:
return -1
return _write_byte(writer, <uint8_t>(0x80 | (utf & 0x3f)))
elif 0xD800 <= utf <= 0xDFFF:
# surogate pair, ignored
return 0
elif utf < 0x10000:
if _write_byte(writer, <uint8_t>(0xe0 | (utf >> 12))) < 0:
return -1
if _write_byte(writer, <uint8_t>(0x80 | ((utf >> 6) & 0x3f))) < 0:
return -1
return _write_byte(writer, <uint8_t>(0x80 | (utf & 0x3f)))
elif utf > 0x10FFFF:
# symbol is too large
return 0
else:
if _write_byte(writer, <uint8_t>(0xf0 | (utf >> 18))) < 0:
return -1
if _write_byte(writer,
<uint8_t>(0x80 | ((utf >> 12) & 0x3f))) < 0:
return -1
if _write_byte(writer,
<uint8_t>(0x80 | ((utf >> 6) & 0x3f))) < 0:
return -1
return _write_byte(writer, <uint8_t>(0x80 | (utf & 0x3f)))
cdef inline int _write_str(Writer* writer, str s):
cdef Py_UCS4 ch
for ch in s:
if _write_utf8(writer, ch) < 0:
return -1
cdef inline int _write_str_raise_on_nlcr(Writer* writer, object s):
cdef Py_UCS4 ch
cdef str out_str
if type(s) is str:
out_str = <str>s
elif type(s) is _istr:
out_str = PyObject_Str(s)
elif not isinstance(s, str):
raise TypeError("Cannot serialize non-str key {!r}".format(s))
else:
out_str = str(s)
for ch in out_str:
# https://www.rfc-editor.org/info/rfc9110/#section-5.5-5
# https://www.rfc-editor.org/info/rfc9112/#section-4-3
if (ch < 0x20 and ch != 0x09) or ch == 0x7F:
raise ValueError(
"Forbidden control character detected in headers. "
"Potential header injection attack."
)
if _write_utf8(writer, ch) < 0:
return -1
# --------------- _serialize_headers ----------------------
def _serialize_headers(str status_line, headers):
cdef Writer writer
cdef object key
cdef object val
cdef char buf[BUF_SIZE]
_init_writer(&writer, buf)
try:
if _write_str_raise_on_nlcr(&writer, status_line) < 0:
raise
if _write_byte(&writer, b'\r') < 0:
raise
if _write_byte(&writer, b'\n') < 0:
raise
for key, val in headers.items():
if _write_str_raise_on_nlcr(&writer, key) < 0:
raise
if _write_byte(&writer, b':') < 0:
raise
if _write_byte(&writer, b' ') < 0:
raise
if _write_str_raise_on_nlcr(&writer, val) < 0:
raise
if _write_byte(&writer, b'\r') < 0:
raise
if _write_byte(&writer, b'\n') < 0:
raise
if _write_byte(&writer, b'\r') < 0:
raise
if _write_byte(&writer, b'\n') < 0:
raise
return PyBytes_FromStringAndSize(writer.buf, writer.pos)
finally:
_release_writer(&writer)
@@ -0,0 +1 @@
"""WebSocket protocol versions 13 and 8."""
+148
View File
@@ -0,0 +1,148 @@
"""Helpers for WebSocket protocol versions 13 and 8."""
import functools
import re
from re import Pattern
from struct import Struct
from typing import TYPE_CHECKING, Final
from ..helpers import NO_EXTENSIONS
from .models import WSHandshakeError
UNPACK_LEN3 = Struct("!Q").unpack_from
UNPACK_CLOSE_CODE = Struct("!H").unpack
PACK_LEN1 = Struct("!BB").pack
PACK_LEN2 = Struct("!BBH").pack
PACK_LEN3 = Struct("!BBQ").pack
PACK_CLOSE_CODE = Struct("!H").pack
PACK_RANDBITS = Struct("!L").pack
MSG_SIZE: Final[int] = 2**14
MASK_LEN: Final[int] = 4
WS_KEY: Final[bytes] = b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
# Used by _websocket_mask_python
@functools.lru_cache
def _xor_table() -> list[bytes]:
return [bytes(a ^ b for a in range(256)) for b in range(256)]
def _websocket_mask_python(mask: bytes, data: bytearray) -> None:
"""Websocket masking function.
`mask` is a `bytes` object of length 4; `data` is a `bytearray`
object of any length. The contents of `data` are masked with `mask`,
as specified in section 5.3 of RFC 6455.
Note that this function mutates the `data` argument.
This pure-python implementation may be replaced by an optimized
version when available.
"""
assert isinstance(data, bytearray), data
assert len(mask) == 4, mask
if data:
_XOR_TABLE = _xor_table()
a, b, c, d = (_XOR_TABLE[n] for n in mask)
data[::4] = data[::4].translate(a)
data[1::4] = data[1::4].translate(b)
data[2::4] = data[2::4].translate(c)
data[3::4] = data[3::4].translate(d)
if TYPE_CHECKING or NO_EXTENSIONS: # pragma: no cover
websocket_mask = _websocket_mask_python
else:
try:
from .mask import _websocket_mask_cython # type: ignore[import-not-found]
websocket_mask = _websocket_mask_cython
except ImportError: # pragma: no cover
websocket_mask = _websocket_mask_python
_WS_EXT_RE: Final[Pattern[str]] = re.compile(
r"^(?:;\s*(?:"
r"(server_no_context_takeover)|"
r"(client_no_context_takeover)|"
r"(server_max_window_bits(?:=(\d+))?)|"
r"(client_max_window_bits(?:=(\d+))?)))*$"
)
_WS_EXT_RE_SPLIT: Final[Pattern[str]] = re.compile(r"permessage-deflate([^,]+)?")
def ws_ext_parse(extstr: str | None, isserver: bool = False) -> tuple[int, bool]:
if not extstr:
return 0, False
compress = 0
notakeover = False
for ext in _WS_EXT_RE_SPLIT.finditer(extstr):
defext = ext.group(1)
# Return compress = 15 when get `permessage-deflate`
if not defext:
compress = 15
break
match = _WS_EXT_RE.match(defext)
if match:
compress = 15
if isserver:
# Server never fail to detect compress handshake.
# Server does not need to send max wbit to client
if match.group(4):
compress = int(match.group(4))
# Group3 must match if group4 matches
# Compress wbit 8 does not support in zlib
# If compress level not support,
# CONTINUE to next extension
if compress > 15 or compress < 9:
compress = 0
continue
if match.group(1):
notakeover = True
# Ignore regex group 5 & 6 for client_max_window_bits
break
else:
if match.group(6):
compress = int(match.group(6))
# Group5 must match if group6 matches
# Compress wbit 8 does not support in zlib
# If compress level not support,
# FAIL the parse progress
if compress > 15 or compress < 9:
raise WSHandshakeError("Invalid window size")
if match.group(2):
notakeover = True
# Ignore regex group 5 & 6 for client_max_window_bits
break
# Return Fail if client side and not match
elif not isserver:
raise WSHandshakeError("Extension for deflate not supported" + ext.group(1))
return compress, notakeover
def ws_ext_gen(
compress: int = 15, isserver: bool = False, server_notakeover: bool = False
) -> str:
# client_notakeover=False not used for server
# compress wbit 8 does not support in zlib
if compress < 9 or compress > 15:
raise ValueError(
"Compress wbits must between 9 and 15, zlib does not support wbits=8"
)
enabledext = ["permessage-deflate"]
if not isserver:
enabledext.append("client_max_window_bits")
if compress < 15:
enabledext.append("server_max_window_bits=" + str(compress))
if server_notakeover:
enabledext.append("server_no_context_takeover")
# if client_notakeover:
# enabledext.append('client_no_context_takeover')
return "; ".join(enabledext)
Binary file not shown.
+3
View File
@@ -0,0 +1,3 @@
"""Cython declarations for websocket masking."""
cpdef void _websocket_mask_cython(bytes mask, bytearray data)
+48
View File
@@ -0,0 +1,48 @@
from cpython cimport PyBytes_AsString
#from cpython cimport PyByteArray_AsString # cython still not exports that
cdef extern from "Python.h":
char* PyByteArray_AsString(bytearray ba) except NULL
from libc.stdint cimport uint32_t, uint64_t, uintmax_t
cpdef void _websocket_mask_cython(bytes mask, bytearray data):
"""Note, this function mutates its `data` argument
"""
cdef:
Py_ssize_t data_len, i
# bit operations on signed integers are implementation-specific
unsigned char * in_buf
const unsigned char * mask_buf
uint32_t uint32_msk
uint64_t uint64_msk
assert len(mask) == 4
data_len = len(data)
in_buf = <unsigned char*>PyByteArray_AsString(data)
mask_buf = <const unsigned char*>PyBytes_AsString(mask)
uint32_msk = (<uint32_t*>mask_buf)[0]
# TODO: align in_data ptr to achieve even faster speeds
# does it need in python ?! malloc() always aligns to sizeof(long) bytes
if sizeof(size_t) >= 8:
uint64_msk = uint32_msk
uint64_msk = (uint64_msk << 32) | uint32_msk
while data_len >= 8:
(<uint64_t*>in_buf)[0] ^= uint64_msk
in_buf += 8
data_len -= 8
while data_len >= 4:
(<uint32_t*>in_buf)[0] ^= uint32_msk
in_buf += 4
data_len -= 4
for i in range(0, data_len):
in_buf[i] ^= mask_buf[i]
+107
View File
@@ -0,0 +1,107 @@
"""Models for WebSocket protocol versions 13 and 8."""
import json
from collections.abc import Callable
from enum import IntEnum
from typing import Any, Final, NamedTuple, cast
WS_DEFLATE_TRAILING: Final[bytes] = bytes([0x00, 0x00, 0xFF, 0xFF])
class WSCloseCode(IntEnum):
OK = 1000
GOING_AWAY = 1001
PROTOCOL_ERROR = 1002
UNSUPPORTED_DATA = 1003
ABNORMAL_CLOSURE = 1006
INVALID_TEXT = 1007
POLICY_VIOLATION = 1008
MESSAGE_TOO_BIG = 1009
MANDATORY_EXTENSION = 1010
INTERNAL_ERROR = 1011
SERVICE_RESTART = 1012
TRY_AGAIN_LATER = 1013
BAD_GATEWAY = 1014
class WSMsgType(IntEnum):
# websocket spec types
CONTINUATION = 0x0
TEXT = 0x1
BINARY = 0x2
PING = 0x9
PONG = 0xA
CLOSE = 0x8
# aiohttp specific types
CLOSING = 0x100
CLOSED = 0x101
ERROR = 0x102
text = TEXT
binary = BINARY
ping = PING
pong = PONG
close = CLOSE
closing = CLOSING
closed = CLOSED
error = ERROR
class WSMessage(NamedTuple):
type: WSMsgType
# To type correctly, this would need some kind of tagged union for each type.
data: Any
extra: str | None
def json(self, *, loads: Callable[[Any], Any] = json.loads) -> Any:
"""Return parsed JSON data.
.. versionadded:: 0.22
"""
return loads(self.data)
class WSMessageTextBytes(NamedTuple):
"""WebSocket TEXT message with raw bytes (no UTF-8 decoding)."""
type: WSMsgType
# To type correctly, this would need some kind of tagged union for each type.
# In 4.0, we use a union of message types to properly type data, but in 3.x
# we keep it as Any to avoid a breaking change.
data: Any
extra: str | None
def json(self, *, loads: Callable[[Any], Any] = json.loads) -> Any:
"""Return parsed JSON data."""
return loads(self.data)
# Type aliases for message types based on decode_text setting
# When decode_text=True, TEXT messages have str data (WSMessage)
# When decode_text=False, TEXT messages have bytes data (WSMessageTextBytes)
WSMessageDecodeText = WSMessage
WSMessageNoDecodeText = WSMessage | WSMessageTextBytes
# Constructing the tuple directly to avoid the overhead of
# the lambda and arg processing since NamedTuples are constructed
# with a run time built lambda
# https://github.com/python/cpython/blob/d83fcf8371f2f33c7797bc8f5423a8bca8c46e5c/Lib/collections/__init__.py#L441
WS_CLOSED_MESSAGE = tuple.__new__(WSMessage, (WSMsgType.CLOSED, None, None))
WS_CLOSING_MESSAGE = tuple.__new__(WSMessage, (WSMsgType.CLOSING, None, None))
class WebSocketError(Exception):
"""WebSocket protocol parser error."""
def __init__(self, code: int, message: str) -> None:
self.code = code
super().__init__(code, message)
def __str__(self) -> str:
return cast(str, self.args[1])
class WSHandshakeError(Exception):
"""WebSocket protocol handshake error."""
+31
View File
@@ -0,0 +1,31 @@
"""Reader for WebSocket protocol versions 13 and 8."""
from typing import TYPE_CHECKING
from ..helpers import NO_EXTENSIONS
if TYPE_CHECKING or NO_EXTENSIONS: # pragma: no cover
from .reader_py import (
WebSocketDataQueue as WebSocketDataQueuePython,
WebSocketReader as WebSocketReaderPython,
)
WebSocketReader = WebSocketReaderPython
WebSocketDataQueue = WebSocketDataQueuePython
else:
try:
from .reader_c import ( # type: ignore[import-not-found]
WebSocketDataQueue as WebSocketDataQueueCython,
WebSocketReader as WebSocketReaderCython,
)
WebSocketReader = WebSocketReaderCython
WebSocketDataQueue = WebSocketDataQueueCython
except ImportError: # pragma: no cover
from .reader_py import (
WebSocketDataQueue as WebSocketDataQueuePython,
WebSocketReader as WebSocketReaderPython,
)
WebSocketReader = WebSocketReaderPython
WebSocketDataQueue = WebSocketDataQueuePython
+112
View File
@@ -0,0 +1,112 @@
import cython
from .mask cimport _websocket_mask_cython as websocket_mask
cdef unsigned int READ_HEADER
cdef unsigned int READ_PAYLOAD_LENGTH
cdef unsigned int READ_PAYLOAD_MASK
cdef unsigned int READ_PAYLOAD
cdef int OP_CODE_NOT_SET
cdef int OP_CODE_CONTINUATION
cdef int OP_CODE_TEXT
cdef int OP_CODE_BINARY
cdef int OP_CODE_CLOSE
cdef int OP_CODE_PING
cdef int OP_CODE_PONG
cdef int COMPRESSED_NOT_SET
cdef int COMPRESSED_FALSE
cdef int COMPRESSED_TRUE
cdef object UNPACK_LEN3
cdef object UNPACK_CLOSE_CODE
cdef object TUPLE_NEW
cdef object WSMsgType
cdef object WSMessage
cdef object WSMessageTextBytes
cdef object WS_MSG_TYPE_TEXT
cdef object WS_MSG_TYPE_BINARY
cdef set ALLOWED_CLOSE_CODES
cdef set MESSAGE_TYPES_WITH_CONTENT
cdef tuple EMPTY_FRAME
cdef tuple EMPTY_FRAME_ERROR
cdef class WebSocketDataQueue:
cdef unsigned int _size
cdef public object _protocol
cdef unsigned int _limit
cdef object _loop
cdef bint _eof
cdef object _waiter
cdef object _exception
cdef public object _buffer
cdef object _get_buffer
cdef object _put_buffer
cdef void _release_waiter(self)
cpdef void feed_data(self, object data, unsigned int size)
@cython.locals(size="unsigned int")
cdef _read_from_buffer(self)
cdef class WebSocketReader:
cdef WebSocketDataQueue queue
cdef unsigned int _max_msg_size
cdef bint _decode_text
cdef Exception _exc
cdef bytearray _partial
cdef unsigned int _state
cdef int _opcode
cdef bint _frame_fin
cdef int _frame_opcode
cdef list _payload_fragments
cdef Py_ssize_t _frame_payload_len
cdef bytes _tail
cdef bint _has_mask
cdef bytes _frame_mask
cdef Py_ssize_t _payload_bytes_to_read
cdef unsigned int _payload_len_flag
cdef int _compressed
cdef object _decompressobj
cdef bint _compress
cpdef tuple feed_data(self, object data)
@cython.locals(
is_continuation=bint,
fin=bint,
has_partial=bint,
payload_merged=bytes,
)
cpdef void _handle_frame(self, bint fin, int opcode, object payload, int compressed) except *
@cython.locals(
start_pos=Py_ssize_t,
data_len=Py_ssize_t,
length=Py_ssize_t,
chunk_size=Py_ssize_t,
chunk_len=Py_ssize_t,
data_len=Py_ssize_t,
data_cstr="const unsigned char *",
first_byte="unsigned char",
second_byte="unsigned char",
f_start_pos=Py_ssize_t,
f_end_pos=Py_ssize_t,
has_mask=bint,
fin=bint,
had_fragments=Py_ssize_t,
payload_bytearray=bytearray,
)
cpdef void _feed_data(self, bytes data) except *
+513
View File
@@ -0,0 +1,513 @@
"""Reader for WebSocket protocol versions 13 and 8."""
import asyncio
import builtins
from collections import deque
from typing import Final
from ..base_protocol import BaseProtocol
from ..compression_utils import ZLibDecompressor
from ..helpers import _EXC_SENTINEL, set_exception
from ..streams import EofStream
from .helpers import UNPACK_CLOSE_CODE, UNPACK_LEN3, websocket_mask
from .models import (
WS_DEFLATE_TRAILING,
WebSocketError,
WSCloseCode,
WSMessage,
WSMessageTextBytes,
WSMsgType,
)
ALLOWED_CLOSE_CODES: Final[set[int]] = {int(i) for i in WSCloseCode}
# States for the reader, used to parse the WebSocket frame
# integer values are used so they can be cythonized
READ_HEADER = 1
READ_PAYLOAD_LENGTH = 2
READ_PAYLOAD_MASK = 3
READ_PAYLOAD = 4
WS_MSG_TYPE_BINARY = WSMsgType.BINARY
WS_MSG_TYPE_TEXT = WSMsgType.TEXT
# WSMsgType values unpacked so they can by cythonized to ints
OP_CODE_NOT_SET = -1
OP_CODE_CONTINUATION = WSMsgType.CONTINUATION.value
OP_CODE_TEXT = WSMsgType.TEXT.value
OP_CODE_BINARY = WSMsgType.BINARY.value
OP_CODE_CLOSE = WSMsgType.CLOSE.value
OP_CODE_PING = WSMsgType.PING.value
OP_CODE_PONG = WSMsgType.PONG.value
EMPTY_FRAME_ERROR = (True, b"")
EMPTY_FRAME = (False, b"")
COMPRESSED_NOT_SET = -1
COMPRESSED_FALSE = 0
COMPRESSED_TRUE = 1
TUPLE_NEW = tuple.__new__
cython_int = int # Typed to int in Python, but cython with use a signed int in the pxd
class WebSocketDataQueue:
"""WebSocketDataQueue resumes and pauses an underlying stream.
It is a destination for WebSocket data.
"""
def __init__(
self, protocol: BaseProtocol, limit: int, *, loop: asyncio.AbstractEventLoop
) -> None:
self._size = 0
self._protocol = protocol
self._limit = limit * 2
self._loop = loop
self._eof = False
self._waiter: asyncio.Future[None] | None = None
self._exception: BaseException | None = None
self._buffer: deque[tuple[WSMessage | WSMessageTextBytes, int]] = deque()
self._get_buffer = self._buffer.popleft
self._put_buffer = self._buffer.append
def is_eof(self) -> bool:
return self._eof
def exception(self) -> BaseException | None:
return self._exception
def set_exception(
self,
exc: BaseException,
exc_cause: builtins.BaseException = _EXC_SENTINEL,
) -> None:
self._eof = True
self._exception = exc
if (waiter := self._waiter) is not None:
self._waiter = None
set_exception(waiter, exc, exc_cause)
def _release_waiter(self) -> None:
if (waiter := self._waiter) is None:
return
self._waiter = None
if not waiter.done():
waiter.set_result(None)
def feed_eof(self) -> None:
self._eof = True
self._release_waiter()
self._exception = None # Break cyclic references
def feed_data(
self, data: "WSMessage | WSMessageTextBytes", size: "cython_int"
) -> None:
self._size += size
self._put_buffer((data, size))
self._release_waiter()
if self._size > self._limit and not self._protocol._reading_paused:
self._protocol.pause_reading()
async def read(self) -> WSMessage | WSMessageTextBytes:
if not self._buffer and not self._eof:
assert not self._waiter
self._waiter = self._loop.create_future()
try:
await self._waiter
except (asyncio.CancelledError, asyncio.TimeoutError):
self._waiter = None
raise
return self._read_from_buffer()
def _read_from_buffer(self) -> WSMessage | WSMessageTextBytes:
if self._buffer:
data, size = self._get_buffer()
self._size -= size
if self._size < self._limit and self._protocol._reading_paused:
self._protocol.resume_reading()
return data
if self._exception is not None:
raise self._exception
raise EofStream
class WebSocketReader:
def __init__(
self,
queue: WebSocketDataQueue,
max_msg_size: int,
compress: bool,
decode_text: bool,
) -> None:
self.queue = queue
self._max_msg_size = max_msg_size
self._decode_text = decode_text
self._exc: Exception | None = None
self._partial = bytearray()
self._state = READ_HEADER
self._opcode: int = OP_CODE_NOT_SET
self._frame_fin = False
self._frame_opcode: int = OP_CODE_NOT_SET
self._payload_fragments: list[bytes] = []
self._frame_payload_len = 0
self._tail: bytes = b""
self._has_mask = False
self._frame_mask: bytes | None = None
self._payload_bytes_to_read = 0
self._payload_len_flag = 0
self._compressed: int = COMPRESSED_NOT_SET
self._decompressobj: ZLibDecompressor | None = None
self._compress = compress
def feed_eof(self) -> None:
self.queue.feed_eof()
# data can be bytearray on Windows because proactor event loop uses bytearray
# and asyncio types this to Union[bytes, bytearray, memoryview] so we need
# coerce data to bytes if it is not
def feed_data(self, data: bytes | bytearray | memoryview) -> tuple[bool, bytes]:
if type(data) is not bytes:
data = bytes(data)
if self._exc is not None:
return True, data
try:
self._feed_data(data)
except Exception as exc:
self._exc = exc
set_exception(self.queue, exc)
return EMPTY_FRAME_ERROR
return EMPTY_FRAME
def _handle_frame(
self,
fin: bool,
opcode: int | cython_int, # Union intended: Cython pxd uses C int
payload: bytes | bytearray,
compressed: int | cython_int, # Union intended: Cython pxd uses C int
) -> None:
msg: WSMessage
if opcode in {OP_CODE_TEXT, OP_CODE_BINARY, OP_CODE_CONTINUATION}:
# Validate continuation frames before processing
if opcode == OP_CODE_CONTINUATION and self._opcode == OP_CODE_NOT_SET:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
"Continuation frame for non started message",
)
# load text/binary
if not fin:
# got partial frame payload
if opcode != OP_CODE_CONTINUATION:
self._opcode = opcode
self._partial += payload
return
has_partial = bool(self._partial)
if opcode == OP_CODE_CONTINUATION:
opcode = self._opcode
self._opcode = OP_CODE_NOT_SET
# previous frame was non finished
# we should get continuation opcode
elif has_partial:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
"The opcode in non-fin frame is expected "
f"to be zero, got {opcode!r}",
)
assembled_payload: bytes | bytearray
if has_partial:
assembled_payload = self._partial + payload
self._partial.clear()
else:
assembled_payload = payload
# Decompress process must to be done after all packets
# received.
if compressed:
if not self._decompressobj:
self._decompressobj = ZLibDecompressor(suppress_deflate_header=True)
# XXX: It's possible that the zlib backend (isal is known to
# do this, maybe others too?) will return max_length bytes,
# but internally buffer more data such that the payload is
# >max_length, so we return one extra byte and if we're able
# to do that, then the message is too big.
payload_merged = self._decompressobj.decompress_sync(
assembled_payload + WS_DEFLATE_TRAILING,
(
self._max_msg_size + 1
if self._max_msg_size
else self._max_msg_size
),
)
if self._max_msg_size and len(payload_merged) > self._max_msg_size:
raise WebSocketError(
WSCloseCode.MESSAGE_TOO_BIG,
f"Decompressed message exceeds size limit {self._max_msg_size}",
)
elif type(assembled_payload) is bytes:
payload_merged = assembled_payload
else:
payload_merged = bytes(assembled_payload)
if opcode == OP_CODE_TEXT:
if self._decode_text:
try:
text = payload_merged.decode("utf-8")
except UnicodeDecodeError as exc:
raise WebSocketError(
WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message"
) from exc
# XXX: The Text and Binary messages here can be a performance
# bottleneck, so we use tuple.__new__ to improve performance.
# This is not type safe, but many tests should fail in
# test_client_ws_functional.py if this is wrong.
self.queue.feed_data(
TUPLE_NEW(WSMessage, (WS_MSG_TYPE_TEXT, text, "")),
len(payload_merged),
)
else:
# Return raw bytes for TEXT messages when decode_text=False
self.queue.feed_data(
TUPLE_NEW(
WSMessageTextBytes, (WS_MSG_TYPE_TEXT, payload_merged, "")
),
len(payload_merged),
)
else:
self.queue.feed_data(
TUPLE_NEW(WSMessage, (WS_MSG_TYPE_BINARY, payload_merged, "")),
len(payload_merged),
)
elif opcode == OP_CODE_CLOSE:
if len(payload) >= 2:
close_code = UNPACK_CLOSE_CODE(payload[:2])[0]
if close_code < 3000 and close_code not in ALLOWED_CLOSE_CODES:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
f"Invalid close code: {close_code}",
)
try:
close_message = payload[2:].decode("utf-8")
except UnicodeDecodeError as exc:
raise WebSocketError(
WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message"
) from exc
msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, close_code, close_message))
elif payload:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
f"Invalid close frame: {fin} {opcode} {payload!r}",
)
else:
msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, 0, ""))
self.queue.feed_data(msg, 0)
elif opcode == OP_CODE_PING:
msg = TUPLE_NEW(WSMessage, (WSMsgType.PING, payload, ""))
self.queue.feed_data(msg, len(payload))
elif opcode == OP_CODE_PONG:
msg = TUPLE_NEW(WSMessage, (WSMsgType.PONG, payload, ""))
self.queue.feed_data(msg, len(payload))
else:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR, f"Unexpected opcode={opcode!r}"
)
def _feed_data(self, data: bytes) -> None:
"""Return the next frame from the socket."""
if self._tail:
data, self._tail = self._tail + data, b""
start_pos: int = 0
data_len = len(data)
data_cstr = data
while True:
# read header
if self._state == READ_HEADER:
if data_len - start_pos < 2:
break
first_byte = data_cstr[start_pos]
second_byte = data_cstr[start_pos + 1]
start_pos += 2
fin = (first_byte >> 7) & 1
rsv1 = (first_byte >> 6) & 1
rsv2 = (first_byte >> 5) & 1
rsv3 = (first_byte >> 4) & 1
opcode = first_byte & 0xF
# frame-fin = %x0 ; more frames of this message follow
# / %x1 ; final frame of this message
# frame-rsv1 = %x0 ;
# 1 bit, MUST be 0 unless negotiated otherwise
# frame-rsv2 = %x0 ;
# 1 bit, MUST be 0 unless negotiated otherwise
# frame-rsv3 = %x0 ;
# 1 bit, MUST be 0 unless negotiated otherwise
#
# Remove rsv1 from this test for deflate development
if rsv2 or rsv3 or (rsv1 and not self._compress):
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
"Received frame with non-zero reserved bits",
)
if opcode not in {
OP_CODE_CONTINUATION,
OP_CODE_TEXT,
OP_CODE_BINARY,
OP_CODE_CLOSE,
OP_CODE_PING,
OP_CODE_PONG,
}:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
f"Unexpected opcode={opcode!r}",
)
if opcode > 0x7 and fin == 0:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
"Received fragmented control frame",
)
has_mask = (second_byte >> 7) & 1
length = second_byte & 0x7F
# Control frames MUST have a payload
# length of 125 bytes or less
if opcode > 0x7 and length > 125:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
"Control frame payload cannot be larger than 125 bytes",
)
# Set compress status if last package is FIN
# OR set compress status if this is first fragment
# Raise error if not first fragment with rsv1 = 0x1
if self._frame_fin or self._compressed == COMPRESSED_NOT_SET:
self._compressed = COMPRESSED_TRUE if rsv1 else COMPRESSED_FALSE
elif rsv1:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
"Received frame with non-zero reserved bits",
)
# Control frames (opcode > 0x7) may be interleaved between the
# fragments of a data message.
# https://datatracker.ietf.org/doc/html/rfc6455#section-5.4
if opcode <= 0x7:
self._frame_fin = bool(fin)
self._frame_opcode = opcode
self._has_mask = bool(has_mask)
self._payload_len_flag = length
self._state = READ_PAYLOAD_LENGTH
# read payload length
if self._state == READ_PAYLOAD_LENGTH:
len_flag = self._payload_len_flag
if len_flag == 126:
if data_len - start_pos < 2:
break
first_byte = data_cstr[start_pos]
second_byte = data_cstr[start_pos + 1]
start_pos += 2
self._payload_bytes_to_read = first_byte << 8 | second_byte
elif len_flag > 126:
if data_len - start_pos < 8:
break
self._payload_bytes_to_read = UNPACK_LEN3(data, start_pos)[0]
start_pos += 8
else:
self._payload_bytes_to_read = len_flag
# Reject oversized data frames before buffering any payload
# bytes. Control frames are capped at 125 bytes (checked in
# READ_HEADER) so only text/binary/continuation need this.
if self._max_msg_size and self._frame_opcode in {
OP_CODE_TEXT,
OP_CODE_BINARY,
OP_CODE_CONTINUATION,
}:
projected_size = self._payload_bytes_to_read + len(self._partial)
if projected_size >= self._max_msg_size:
raise WebSocketError(
WSCloseCode.MESSAGE_TOO_BIG,
f"Message size {projected_size} "
f"exceeds limit {self._max_msg_size}",
)
self._state = READ_PAYLOAD_MASK if self._has_mask else READ_PAYLOAD
# read payload mask
if self._state == READ_PAYLOAD_MASK:
if data_len - start_pos < 4:
break
self._frame_mask = data_cstr[start_pos : start_pos + 4]
start_pos += 4
self._state = READ_PAYLOAD
if self._state == READ_PAYLOAD:
chunk_len = data_len - start_pos
if self._payload_bytes_to_read >= chunk_len:
f_end_pos = data_len
self._payload_bytes_to_read -= chunk_len
else:
f_end_pos = start_pos + self._payload_bytes_to_read
self._payload_bytes_to_read = 0
had_fragments = self._frame_payload_len
self._frame_payload_len += f_end_pos - start_pos
f_start_pos = start_pos
start_pos = f_end_pos
if self._payload_bytes_to_read != 0:
# If we don't have a complete frame, we need to save the
# data for the next call to feed_data.
self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos])
break
payload: bytes | bytearray
if had_fragments:
# We have to join the payload fragments get the payload
self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos])
if self._has_mask:
assert self._frame_mask is not None
payload_bytearray = bytearray(b"".join(self._payload_fragments))
websocket_mask(self._frame_mask, payload_bytearray)
payload = payload_bytearray
else:
payload = b"".join(self._payload_fragments)
self._payload_fragments.clear()
elif self._has_mask:
assert self._frame_mask is not None
payload_bytearray = data_cstr[f_start_pos:f_end_pos] # type: ignore[assignment]
if type(payload_bytearray) is not bytearray: # pragma: no branch
# Cython will do the conversion for us
# but we need to do it for Python and we
# will always get here in Python
payload_bytearray = bytearray(payload_bytearray)
websocket_mask(self._frame_mask, payload_bytearray)
payload = payload_bytearray
else:
payload = data_cstr[f_start_pos:f_end_pos]
self._handle_frame(
self._frame_fin, self._frame_opcode, payload, self._compressed
)
self._frame_payload_len = 0
self._state = READ_HEADER
# XXX: Cython needs slices to be bounded, so we can't omit the slice end here.
self._tail = data_cstr[start_pos:data_len] if start_pos < data_len else b""
+513
View File
@@ -0,0 +1,513 @@
"""Reader for WebSocket protocol versions 13 and 8."""
import asyncio
import builtins
from collections import deque
from typing import Final
from ..base_protocol import BaseProtocol
from ..compression_utils import ZLibDecompressor
from ..helpers import _EXC_SENTINEL, set_exception
from ..streams import EofStream
from .helpers import UNPACK_CLOSE_CODE, UNPACK_LEN3, websocket_mask
from .models import (
WS_DEFLATE_TRAILING,
WebSocketError,
WSCloseCode,
WSMessage,
WSMessageTextBytes,
WSMsgType,
)
ALLOWED_CLOSE_CODES: Final[set[int]] = {int(i) for i in WSCloseCode}
# States for the reader, used to parse the WebSocket frame
# integer values are used so they can be cythonized
READ_HEADER = 1
READ_PAYLOAD_LENGTH = 2
READ_PAYLOAD_MASK = 3
READ_PAYLOAD = 4
WS_MSG_TYPE_BINARY = WSMsgType.BINARY
WS_MSG_TYPE_TEXT = WSMsgType.TEXT
# WSMsgType values unpacked so they can by cythonized to ints
OP_CODE_NOT_SET = -1
OP_CODE_CONTINUATION = WSMsgType.CONTINUATION.value
OP_CODE_TEXT = WSMsgType.TEXT.value
OP_CODE_BINARY = WSMsgType.BINARY.value
OP_CODE_CLOSE = WSMsgType.CLOSE.value
OP_CODE_PING = WSMsgType.PING.value
OP_CODE_PONG = WSMsgType.PONG.value
EMPTY_FRAME_ERROR = (True, b"")
EMPTY_FRAME = (False, b"")
COMPRESSED_NOT_SET = -1
COMPRESSED_FALSE = 0
COMPRESSED_TRUE = 1
TUPLE_NEW = tuple.__new__
cython_int = int # Typed to int in Python, but cython with use a signed int in the pxd
class WebSocketDataQueue:
"""WebSocketDataQueue resumes and pauses an underlying stream.
It is a destination for WebSocket data.
"""
def __init__(
self, protocol: BaseProtocol, limit: int, *, loop: asyncio.AbstractEventLoop
) -> None:
self._size = 0
self._protocol = protocol
self._limit = limit * 2
self._loop = loop
self._eof = False
self._waiter: asyncio.Future[None] | None = None
self._exception: BaseException | None = None
self._buffer: deque[tuple[WSMessage | WSMessageTextBytes, int]] = deque()
self._get_buffer = self._buffer.popleft
self._put_buffer = self._buffer.append
def is_eof(self) -> bool:
return self._eof
def exception(self) -> BaseException | None:
return self._exception
def set_exception(
self,
exc: BaseException,
exc_cause: builtins.BaseException = _EXC_SENTINEL,
) -> None:
self._eof = True
self._exception = exc
if (waiter := self._waiter) is not None:
self._waiter = None
set_exception(waiter, exc, exc_cause)
def _release_waiter(self) -> None:
if (waiter := self._waiter) is None:
return
self._waiter = None
if not waiter.done():
waiter.set_result(None)
def feed_eof(self) -> None:
self._eof = True
self._release_waiter()
self._exception = None # Break cyclic references
def feed_data(
self, data: "WSMessage | WSMessageTextBytes", size: "cython_int"
) -> None:
self._size += size
self._put_buffer((data, size))
self._release_waiter()
if self._size > self._limit and not self._protocol._reading_paused:
self._protocol.pause_reading()
async def read(self) -> WSMessage | WSMessageTextBytes:
if not self._buffer and not self._eof:
assert not self._waiter
self._waiter = self._loop.create_future()
try:
await self._waiter
except (asyncio.CancelledError, asyncio.TimeoutError):
self._waiter = None
raise
return self._read_from_buffer()
def _read_from_buffer(self) -> WSMessage | WSMessageTextBytes:
if self._buffer:
data, size = self._get_buffer()
self._size -= size
if self._size < self._limit and self._protocol._reading_paused:
self._protocol.resume_reading()
return data
if self._exception is not None:
raise self._exception
raise EofStream
class WebSocketReader:
def __init__(
self,
queue: WebSocketDataQueue,
max_msg_size: int,
compress: bool,
decode_text: bool,
) -> None:
self.queue = queue
self._max_msg_size = max_msg_size
self._decode_text = decode_text
self._exc: Exception | None = None
self._partial = bytearray()
self._state = READ_HEADER
self._opcode: int = OP_CODE_NOT_SET
self._frame_fin = False
self._frame_opcode: int = OP_CODE_NOT_SET
self._payload_fragments: list[bytes] = []
self._frame_payload_len = 0
self._tail: bytes = b""
self._has_mask = False
self._frame_mask: bytes | None = None
self._payload_bytes_to_read = 0
self._payload_len_flag = 0
self._compressed: int = COMPRESSED_NOT_SET
self._decompressobj: ZLibDecompressor | None = None
self._compress = compress
def feed_eof(self) -> None:
self.queue.feed_eof()
# data can be bytearray on Windows because proactor event loop uses bytearray
# and asyncio types this to Union[bytes, bytearray, memoryview] so we need
# coerce data to bytes if it is not
def feed_data(self, data: bytes | bytearray | memoryview) -> tuple[bool, bytes]:
if type(data) is not bytes:
data = bytes(data)
if self._exc is not None:
return True, data
try:
self._feed_data(data)
except Exception as exc:
self._exc = exc
set_exception(self.queue, exc)
return EMPTY_FRAME_ERROR
return EMPTY_FRAME
def _handle_frame(
self,
fin: bool,
opcode: int | cython_int, # Union intended: Cython pxd uses C int
payload: bytes | bytearray,
compressed: int | cython_int, # Union intended: Cython pxd uses C int
) -> None:
msg: WSMessage
if opcode in {OP_CODE_TEXT, OP_CODE_BINARY, OP_CODE_CONTINUATION}:
# Validate continuation frames before processing
if opcode == OP_CODE_CONTINUATION and self._opcode == OP_CODE_NOT_SET:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
"Continuation frame for non started message",
)
# load text/binary
if not fin:
# got partial frame payload
if opcode != OP_CODE_CONTINUATION:
self._opcode = opcode
self._partial += payload
return
has_partial = bool(self._partial)
if opcode == OP_CODE_CONTINUATION:
opcode = self._opcode
self._opcode = OP_CODE_NOT_SET
# previous frame was non finished
# we should get continuation opcode
elif has_partial:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
"The opcode in non-fin frame is expected "
f"to be zero, got {opcode!r}",
)
assembled_payload: bytes | bytearray
if has_partial:
assembled_payload = self._partial + payload
self._partial.clear()
else:
assembled_payload = payload
# Decompress process must to be done after all packets
# received.
if compressed:
if not self._decompressobj:
self._decompressobj = ZLibDecompressor(suppress_deflate_header=True)
# XXX: It's possible that the zlib backend (isal is known to
# do this, maybe others too?) will return max_length bytes,
# but internally buffer more data such that the payload is
# >max_length, so we return one extra byte and if we're able
# to do that, then the message is too big.
payload_merged = self._decompressobj.decompress_sync(
assembled_payload + WS_DEFLATE_TRAILING,
(
self._max_msg_size + 1
if self._max_msg_size
else self._max_msg_size
),
)
if self._max_msg_size and len(payload_merged) > self._max_msg_size:
raise WebSocketError(
WSCloseCode.MESSAGE_TOO_BIG,
f"Decompressed message exceeds size limit {self._max_msg_size}",
)
elif type(assembled_payload) is bytes:
payload_merged = assembled_payload
else:
payload_merged = bytes(assembled_payload)
if opcode == OP_CODE_TEXT:
if self._decode_text:
try:
text = payload_merged.decode("utf-8")
except UnicodeDecodeError as exc:
raise WebSocketError(
WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message"
) from exc
# XXX: The Text and Binary messages here can be a performance
# bottleneck, so we use tuple.__new__ to improve performance.
# This is not type safe, but many tests should fail in
# test_client_ws_functional.py if this is wrong.
self.queue.feed_data(
TUPLE_NEW(WSMessage, (WS_MSG_TYPE_TEXT, text, "")),
len(payload_merged),
)
else:
# Return raw bytes for TEXT messages when decode_text=False
self.queue.feed_data(
TUPLE_NEW(
WSMessageTextBytes, (WS_MSG_TYPE_TEXT, payload_merged, "")
),
len(payload_merged),
)
else:
self.queue.feed_data(
TUPLE_NEW(WSMessage, (WS_MSG_TYPE_BINARY, payload_merged, "")),
len(payload_merged),
)
elif opcode == OP_CODE_CLOSE:
if len(payload) >= 2:
close_code = UNPACK_CLOSE_CODE(payload[:2])[0]
if close_code < 3000 and close_code not in ALLOWED_CLOSE_CODES:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
f"Invalid close code: {close_code}",
)
try:
close_message = payload[2:].decode("utf-8")
except UnicodeDecodeError as exc:
raise WebSocketError(
WSCloseCode.INVALID_TEXT, "Invalid UTF-8 text message"
) from exc
msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, close_code, close_message))
elif payload:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
f"Invalid close frame: {fin} {opcode} {payload!r}",
)
else:
msg = TUPLE_NEW(WSMessage, (WSMsgType.CLOSE, 0, ""))
self.queue.feed_data(msg, 0)
elif opcode == OP_CODE_PING:
msg = TUPLE_NEW(WSMessage, (WSMsgType.PING, payload, ""))
self.queue.feed_data(msg, len(payload))
elif opcode == OP_CODE_PONG:
msg = TUPLE_NEW(WSMessage, (WSMsgType.PONG, payload, ""))
self.queue.feed_data(msg, len(payload))
else:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR, f"Unexpected opcode={opcode!r}"
)
def _feed_data(self, data: bytes) -> None:
"""Return the next frame from the socket."""
if self._tail:
data, self._tail = self._tail + data, b""
start_pos: int = 0
data_len = len(data)
data_cstr = data
while True:
# read header
if self._state == READ_HEADER:
if data_len - start_pos < 2:
break
first_byte = data_cstr[start_pos]
second_byte = data_cstr[start_pos + 1]
start_pos += 2
fin = (first_byte >> 7) & 1
rsv1 = (first_byte >> 6) & 1
rsv2 = (first_byte >> 5) & 1
rsv3 = (first_byte >> 4) & 1
opcode = first_byte & 0xF
# frame-fin = %x0 ; more frames of this message follow
# / %x1 ; final frame of this message
# frame-rsv1 = %x0 ;
# 1 bit, MUST be 0 unless negotiated otherwise
# frame-rsv2 = %x0 ;
# 1 bit, MUST be 0 unless negotiated otherwise
# frame-rsv3 = %x0 ;
# 1 bit, MUST be 0 unless negotiated otherwise
#
# Remove rsv1 from this test for deflate development
if rsv2 or rsv3 or (rsv1 and not self._compress):
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
"Received frame with non-zero reserved bits",
)
if opcode not in {
OP_CODE_CONTINUATION,
OP_CODE_TEXT,
OP_CODE_BINARY,
OP_CODE_CLOSE,
OP_CODE_PING,
OP_CODE_PONG,
}:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
f"Unexpected opcode={opcode!r}",
)
if opcode > 0x7 and fin == 0:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
"Received fragmented control frame",
)
has_mask = (second_byte >> 7) & 1
length = second_byte & 0x7F
# Control frames MUST have a payload
# length of 125 bytes or less
if opcode > 0x7 and length > 125:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
"Control frame payload cannot be larger than 125 bytes",
)
# Set compress status if last package is FIN
# OR set compress status if this is first fragment
# Raise error if not first fragment with rsv1 = 0x1
if self._frame_fin or self._compressed == COMPRESSED_NOT_SET:
self._compressed = COMPRESSED_TRUE if rsv1 else COMPRESSED_FALSE
elif rsv1:
raise WebSocketError(
WSCloseCode.PROTOCOL_ERROR,
"Received frame with non-zero reserved bits",
)
# Control frames (opcode > 0x7) may be interleaved between the
# fragments of a data message.
# https://datatracker.ietf.org/doc/html/rfc6455#section-5.4
if opcode <= 0x7:
self._frame_fin = bool(fin)
self._frame_opcode = opcode
self._has_mask = bool(has_mask)
self._payload_len_flag = length
self._state = READ_PAYLOAD_LENGTH
# read payload length
if self._state == READ_PAYLOAD_LENGTH:
len_flag = self._payload_len_flag
if len_flag == 126:
if data_len - start_pos < 2:
break
first_byte = data_cstr[start_pos]
second_byte = data_cstr[start_pos + 1]
start_pos += 2
self._payload_bytes_to_read = first_byte << 8 | second_byte
elif len_flag > 126:
if data_len - start_pos < 8:
break
self._payload_bytes_to_read = UNPACK_LEN3(data, start_pos)[0]
start_pos += 8
else:
self._payload_bytes_to_read = len_flag
# Reject oversized data frames before buffering any payload
# bytes. Control frames are capped at 125 bytes (checked in
# READ_HEADER) so only text/binary/continuation need this.
if self._max_msg_size and self._frame_opcode in {
OP_CODE_TEXT,
OP_CODE_BINARY,
OP_CODE_CONTINUATION,
}:
projected_size = self._payload_bytes_to_read + len(self._partial)
if projected_size >= self._max_msg_size:
raise WebSocketError(
WSCloseCode.MESSAGE_TOO_BIG,
f"Message size {projected_size} "
f"exceeds limit {self._max_msg_size}",
)
self._state = READ_PAYLOAD_MASK if self._has_mask else READ_PAYLOAD
# read payload mask
if self._state == READ_PAYLOAD_MASK:
if data_len - start_pos < 4:
break
self._frame_mask = data_cstr[start_pos : start_pos + 4]
start_pos += 4
self._state = READ_PAYLOAD
if self._state == READ_PAYLOAD:
chunk_len = data_len - start_pos
if self._payload_bytes_to_read >= chunk_len:
f_end_pos = data_len
self._payload_bytes_to_read -= chunk_len
else:
f_end_pos = start_pos + self._payload_bytes_to_read
self._payload_bytes_to_read = 0
had_fragments = self._frame_payload_len
self._frame_payload_len += f_end_pos - start_pos
f_start_pos = start_pos
start_pos = f_end_pos
if self._payload_bytes_to_read != 0:
# If we don't have a complete frame, we need to save the
# data for the next call to feed_data.
self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos])
break
payload: bytes | bytearray
if had_fragments:
# We have to join the payload fragments get the payload
self._payload_fragments.append(data_cstr[f_start_pos:f_end_pos])
if self._has_mask:
assert self._frame_mask is not None
payload_bytearray = bytearray(b"".join(self._payload_fragments))
websocket_mask(self._frame_mask, payload_bytearray)
payload = payload_bytearray
else:
payload = b"".join(self._payload_fragments)
self._payload_fragments.clear()
elif self._has_mask:
assert self._frame_mask is not None
payload_bytearray = data_cstr[f_start_pos:f_end_pos] # type: ignore[assignment]
if type(payload_bytearray) is not bytearray: # pragma: no branch
# Cython will do the conversion for us
# but we need to do it for Python and we
# will always get here in Python
payload_bytearray = bytearray(payload_bytearray)
websocket_mask(self._frame_mask, payload_bytearray)
payload = payload_bytearray
else:
payload = data_cstr[f_start_pos:f_end_pos]
self._handle_frame(
self._frame_fin, self._frame_opcode, payload, self._compressed
)
self._frame_payload_len = 0
self._state = READ_HEADER
# XXX: Cython needs slices to be bounded, so we can't omit the slice end here.
self._tail = data_cstr[start_pos:data_len] if start_pos < data_len else b""
+261
View File
@@ -0,0 +1,261 @@
"""WebSocket protocol versions 13 and 8."""
import asyncio
import random
import sys
from functools import partial
from typing import Final, Optional, Set
from ..base_protocol import BaseProtocol
from ..client_exceptions import ClientConnectionResetError
from ..compression_utils import ZLibBackend, ZLibCompressor
from ..helpers import DEFAULT_CHUNK_SIZE
from .helpers import (
MASK_LEN,
MSG_SIZE,
PACK_CLOSE_CODE,
PACK_LEN1,
PACK_LEN2,
PACK_LEN3,
PACK_RANDBITS,
websocket_mask,
)
from .models import WS_DEFLATE_TRAILING, WSMsgType
# WebSocket opcode boundary: opcodes 0-7 are data frames, 8-15 are control frames
# Control frames (ping, pong, close) are never compressed
WS_CONTROL_FRAME_OPCODE: Final[int] = 8
# For websockets, keeping latency low is extremely important as implementations
# generally expect to be able to send and receive messages quickly. We use a
# larger chunk size to reduce the number of executor calls and avoid task
# creation overhead, since both are significant sources of latency when chunks
# are small. A size of 16KiB was chosen as a balance between avoiding task
# overhead and not blocking the event loop too long with synchronous compression.
WEBSOCKET_MAX_SYNC_CHUNK_SIZE = 16 * 1024
class WebSocketWriter:
"""WebSocket writer.
The writer is responsible for sending messages to the client. It is
created by the protocol when a connection is established. The writer
should avoid implementing any application logic and should only be
concerned with the low-level details of the WebSocket protocol.
"""
def __init__(
self,
protocol: BaseProtocol,
transport: asyncio.Transport,
*,
use_mask: bool = False,
limit: int = DEFAULT_CHUNK_SIZE,
random: random.Random = random.Random(),
compress: int = 0,
notakeover: bool = False,
) -> None:
"""Initialize a WebSocket writer."""
self.protocol = protocol
self.transport = transport
self.use_mask = use_mask
self.get_random_bits = partial(random.getrandbits, 32)
self.compress = compress
self.notakeover = notakeover
self._closing = False
self._limit = limit
self._output_size = 0
self._compressobj: Optional[ZLibCompressor] = None
self._send_lock = asyncio.Lock()
self._background_tasks: Set[asyncio.Task[None]] = set()
async def send_frame(
self, message: bytes, opcode: int, compress: int | None = None
) -> None:
"""Send a frame over the websocket with message as its payload."""
if self._closing and not (opcode & WSMsgType.CLOSE):
raise ClientConnectionResetError("Cannot write to closing transport")
if not (compress or self.compress) or opcode >= WS_CONTROL_FRAME_OPCODE:
# Non-compressed frames don't need lock or shield
self._write_websocket_frame(message, opcode, 0)
elif len(message) <= WEBSOCKET_MAX_SYNC_CHUNK_SIZE:
# Small compressed payloads - compress synchronously in event loop
# We need the lock even though sync compression has no await points.
# This prevents small frames from interleaving with large frames that
# compress in the executor, avoiding compressor state corruption.
async with self._send_lock:
self._send_compressed_frame_sync(message, opcode, compress)
else:
# Large compressed frames need shield to prevent corruption
# For large compressed frames, the entire compress+send
# operation must be atomic. If cancelled after compression but
# before send, the compressor state would be advanced but data
# not sent, corrupting subsequent frames.
# Create a task to shield from cancellation
# The lock is acquired inside the shielded task so the entire
# operation (lock + compress + send) completes atomically.
# Use eager_start on Python 3.12+ to avoid scheduling overhead
loop = asyncio.get_running_loop()
coro = self._send_compressed_frame_async_locked(message, opcode, compress)
if sys.version_info >= (3, 12):
send_task = asyncio.Task(coro, loop=loop, eager_start=True)
else:
send_task = loop.create_task(coro)
# Keep a strong reference to prevent garbage collection
self._background_tasks.add(send_task)
send_task.add_done_callback(self._background_tasks.discard)
await asyncio.shield(send_task)
# It is safe to return control to the event loop when using compression
# after this point as we have already sent or buffered all the data.
# Once we have written output_size up to the limit, we call the
# drain helper which waits for the transport to be ready to accept
# more data. This is a flow control mechanism to prevent the buffer
# from growing too large. The drain helper will return right away
# if the writer is not paused.
if self._output_size > self._limit:
self._output_size = 0
if self.protocol._paused:
await self.protocol._drain_helper()
def _write_websocket_frame(self, message: bytes, opcode: int, rsv: int) -> None:
"""
Write a websocket frame to the transport.
This method handles frame header construction, masking, and writing to transport.
It does not handle compression or flow control - those are the responsibility
of the caller.
"""
msg_length = len(message)
use_mask = self.use_mask
mask_bit = 0x80 if use_mask else 0
# Depending on the message length, the header is assembled differently.
# The first byte is reserved for the opcode and the RSV bits.
first_byte = 0x80 | rsv | opcode
if msg_length < 126:
header = PACK_LEN1(first_byte, msg_length | mask_bit)
header_len = 2
elif msg_length < 65536:
header = PACK_LEN2(first_byte, 126 | mask_bit, msg_length)
header_len = 4
else:
header = PACK_LEN3(first_byte, 127 | mask_bit, msg_length)
header_len = 10
if self.transport.is_closing():
raise ClientConnectionResetError("Cannot write to closing transport")
# https://datatracker.ietf.org/doc/html/rfc6455#section-5.3
# If we are using a mask, we need to generate it randomly
# and apply it to the message before sending it. A mask is
# a 32-bit value that is applied to the message using a
# bitwise XOR operation. It is used to prevent certain types
# of attacks on the websocket protocol. The mask is only used
# when aiohttp is acting as a client. Servers do not use a mask.
if use_mask:
mask = PACK_RANDBITS(self.get_random_bits())
message_arr = bytearray(message)
websocket_mask(mask, message_arr)
self.transport.write(header + mask + message_arr)
self._output_size += MASK_LEN
elif msg_length > MSG_SIZE:
self.transport.write(header)
self.transport.write(message)
else:
self.transport.write(header + message)
self._output_size += header_len + msg_length
def _get_compressor(self, compress: int | None) -> ZLibCompressor:
"""Get or create a compressor object for the given compression level."""
if compress:
# Do not set self._compress if compressing is for this frame
return ZLibCompressor(
level=ZLibBackend.Z_BEST_SPEED,
wbits=-compress,
max_sync_chunk_size=WEBSOCKET_MAX_SYNC_CHUNK_SIZE,
)
if not self._compressobj:
self._compressobj = ZLibCompressor(
level=ZLibBackend.Z_BEST_SPEED,
wbits=-self.compress,
max_sync_chunk_size=WEBSOCKET_MAX_SYNC_CHUNK_SIZE,
)
return self._compressobj
def _send_compressed_frame_sync(
self, message: bytes, opcode: int, compress: int | None
) -> None:
"""
Synchronous send for small compressed frames.
This is used for small compressed payloads that compress synchronously in the event loop.
Since there are no await points, this is inherently cancellation-safe.
"""
# RSV are the reserved bits in the frame header. They are used to
# indicate that the frame is using an extension.
# https://datatracker.ietf.org/doc/html/rfc6455#section-5.2
compressobj = self._get_compressor(compress)
# (0x40) RSV1 is set for compressed frames
# https://datatracker.ietf.org/doc/html/rfc7692#section-7.2.3.1
self._write_websocket_frame(
(
compressobj.compress_sync(message)
+ compressobj.flush(
ZLibBackend.Z_FULL_FLUSH
if self.notakeover
else ZLibBackend.Z_SYNC_FLUSH
)
).removesuffix(WS_DEFLATE_TRAILING),
opcode,
0x40,
)
async def _send_compressed_frame_async_locked(
self, message: bytes, opcode: int, compress: int | None
) -> None:
"""
Async send for large compressed frames with lock.
Acquires the lock and compresses large payloads asynchronously in
the executor. The lock is held for the entire operation to ensure
the compressor state is not corrupted by concurrent sends.
MUST be run shielded from cancellation. If cancelled after
compression but before sending, the compressor state would be
advanced but data not sent, corrupting subsequent frames.
"""
async with self._send_lock:
# RSV are the reserved bits in the frame header. They are used to
# indicate that the frame is using an extension.
# https://datatracker.ietf.org/doc/html/rfc6455#section-5.2
compressobj = self._get_compressor(compress)
# (0x40) RSV1 is set for compressed frames
# https://datatracker.ietf.org/doc/html/rfc7692#section-7.2.3.1
self._write_websocket_frame(
(
await compressobj.compress(message)
+ compressobj.flush(
ZLibBackend.Z_FULL_FLUSH
if self.notakeover
else ZLibBackend.Z_SYNC_FLUSH
)
).removesuffix(WS_DEFLATE_TRAILING),
opcode,
0x40,
)
async def close(self, code: int = 1000, message: bytes | str = b"") -> None:
"""Close the websocket, sending the specified code and message."""
if isinstance(message, str):
message = message.encode("utf-8")
try:
await self.send_frame(
PACK_CLOSE_CODE(code) + message, opcode=WSMsgType.CLOSE
)
finally:
self._closing = True
+270
View File
@@ -0,0 +1,270 @@
import asyncio
import logging
import socket
from abc import ABC, abstractmethod
from collections.abc import Awaitable, Callable, Generator, Iterable, Sequence, Sized
from http.cookies import BaseCookie, Morsel, SimpleCookie
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, TypedDict
from multidict import CIMultiDict
from yarl import URL
from ._cookie_helpers import parse_set_cookie_headers
from .typedefs import LooseCookies
if TYPE_CHECKING:
from .web_app import Application
from .web_exceptions import HTTPException
from .web_request import BaseRequest, Request
from .web_response import StreamResponse
else:
BaseRequest = Request = Application = StreamResponse = Any
HTTPException = Any
class AbstractRouter(ABC):
def __init__(self) -> None:
self._frozen = False
def post_init(self, app: Application) -> None:
"""Post init stage.
Not an abstract method for sake of backward compatibility,
but if the router wants to be aware of the application
it can override this.
"""
@property
def frozen(self) -> bool:
return self._frozen
def freeze(self) -> None:
"""Freeze router."""
self._frozen = True
@abstractmethod
async def resolve(self, request: Request) -> "AbstractMatchInfo":
"""Return MATCH_INFO for given request"""
class AbstractMatchInfo(ABC):
__slots__ = ()
@property # pragma: no branch
@abstractmethod
def handler(self) -> Callable[[Request], Awaitable[StreamResponse]]:
"""Execute matched request handler"""
@property
@abstractmethod
def expect_handler(
self,
) -> Callable[[Request], Awaitable[StreamResponse | None]]:
"""Expect handler for 100-continue processing"""
@property # pragma: no branch
@abstractmethod
def http_exception(self) -> HTTPException | None:
"""HTTPException instance raised on router's resolving, or None"""
@abstractmethod # pragma: no branch
def get_info(self) -> dict[str, Any]:
"""Return a dict with additional info useful for introspection"""
@property # pragma: no branch
@abstractmethod
def apps(self) -> tuple[Application, ...]:
"""Stack of nested applications.
Top level application is left-most element.
"""
@abstractmethod
def add_app(self, app: Application) -> None:
"""Add application to the nested apps stack."""
@abstractmethod
def freeze(self) -> None:
"""Freeze the match info.
The method is called after route resolution.
After the call .add_app() is forbidden.
"""
class AbstractView(ABC):
"""Abstract class based view."""
def __init__(self, request: Request) -> None:
self._request = request
@property
def request(self) -> Request:
"""Request instance."""
return self._request
@abstractmethod
def __await__(self) -> Generator[None, None, StreamResponse]:
"""Execute the view handler."""
class ResolveResult(TypedDict):
"""Resolve result.
This is the result returned from an AbstractResolver's
resolve method.
:param hostname: The hostname that was provided.
:param host: The IP address that was resolved.
:param port: The port that was resolved.
:param family: The address family that was resolved.
:param proto: The protocol that was resolved.
:param flags: The flags that were resolved.
"""
hostname: str
host: str
port: int
family: int
proto: int
flags: int
class AbstractResolver(ABC):
"""Abstract DNS resolver."""
@abstractmethod
async def resolve(
self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET
) -> list[ResolveResult]:
"""Return IP address for given hostname"""
@abstractmethod
async def close(self) -> None:
"""Release resolver"""
if TYPE_CHECKING:
IterableBase = Iterable[Morsel[str]]
else:
IterableBase = Iterable
ClearCookiePredicate = Callable[["Morsel[str]"], bool]
class AbstractCookieJar(Sized, IterableBase):
"""Abstract Cookie Jar."""
def __init__(self, *, loop: asyncio.AbstractEventLoop | None = None) -> None:
self._loop = loop or asyncio.get_running_loop()
@property
@abstractmethod
def unsafe(self) -> bool:
"""Return True if cookies can be used with IP addresses."""
@property
@abstractmethod
def quote_cookie(self) -> bool:
"""Return True if cookies should be quoted."""
@property
@abstractmethod
def cookies(self) -> MappingProxyType[tuple[str, str], SimpleCookie]:
"""Return the cookies stored in this jar."""
@property
@abstractmethod
def host_only_cookies(self) -> frozenset[tuple[str, str]]:
"""Return the host-only cookies stored in this jar."""
@abstractmethod
def clear(self, predicate: ClearCookiePredicate | None = None) -> None:
"""Clear all cookies if no predicate is passed."""
@abstractmethod
def clear_domain(self, domain: str) -> None:
"""Clear all cookies for domain and all subdomains."""
@abstractmethod
def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None:
"""Update cookies."""
def update_cookies_from_headers(
self, headers: Sequence[str], response_url: URL
) -> None:
"""Update cookies from raw Set-Cookie headers."""
if headers and (cookies_to_update := parse_set_cookie_headers(headers)):
self.update_cookies(cookies_to_update, response_url)
@abstractmethod
def filter_cookies(self, request_url: URL) -> "BaseCookie[str]":
"""Return the jar's cookies filtered by their attributes."""
class AbstractStreamWriter(ABC):
"""Abstract stream writer."""
buffer_size: int = 0
output_size: int = 0
length: int | None = 0
@abstractmethod
async def write(self, chunk: bytes | bytearray | memoryview) -> None:
"""Write chunk into stream."""
@abstractmethod
async def write_eof(self, chunk: bytes = b"") -> None:
"""Write last chunk."""
@abstractmethod
async def drain(self) -> None:
"""Flush the write buffer."""
@abstractmethod
def enable_compression(
self, encoding: str = "deflate", strategy: int | None = None
) -> None:
"""Enable HTTP body compression"""
@abstractmethod
def enable_chunking(self) -> None:
"""Enable HTTP chunked mode"""
@abstractmethod
async def write_headers(
self, status_line: str, headers: "CIMultiDict[str]"
) -> None:
"""Write HTTP headers"""
def send_headers(self) -> None:
"""Force sending buffered headers if not already sent.
Required only if write_headers() buffers headers instead of sending immediately.
For backwards compatibility, this method does nothing by default.
"""
class AbstractAccessLogger(ABC):
"""Abstract writer to access log."""
__slots__ = ("logger", "log_format")
def __init__(self, logger: logging.Logger, log_format: str) -> None:
self.logger = logger
self.log_format = log_format
@abstractmethod
def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
"""Emit log to logger."""
@property
def enabled(self) -> bool:
"""Check if logger is enabled."""
return True
+140
View File
@@ -0,0 +1,140 @@
import asyncio
from typing import TYPE_CHECKING, Any, cast
from .client_exceptions import ClientConnectionResetError
from .helpers import set_exception
from .tcp_helpers import tcp_nodelay
if TYPE_CHECKING:
from .http_parser import HttpParser
# Raised by transport.pause_reading()/resume_reading() when the transport
# does not support flow control; safe to ignore.
# NOTE: Catch these with a plain try/except/pass, never contextlib.suppress():
# pause/resume run on the hot read path and suppress() is ~6x slower than
# try/except here (it builds a context manager and unpacks this tuple per call).
PAUSE_RESUME_READING_ERRORS = (AttributeError, NotImplementedError, RuntimeError)
class BaseProtocol(asyncio.Protocol):
__slots__ = (
"_loop",
"_paused",
"_parser",
"_drain_waiter",
"_connection_lost",
"_reading_paused",
"_upgraded",
"transport",
)
def __init__(
self, loop: asyncio.AbstractEventLoop, parser: "HttpParser[Any] | None" = None
) -> None:
self._loop: asyncio.AbstractEventLoop = loop
self._paused = False
self._drain_waiter: asyncio.Future[None] | None = None
self._reading_paused = False
self._parser = parser
self._upgraded = False
self.transport: asyncio.Transport | None = None
@property
def connected(self) -> bool:
"""Return True if the connection is open."""
return self.transport is not None
@property
def writing_paused(self) -> bool:
return self._paused
def pause_writing(self) -> None:
assert not self._paused
self._paused = True
def resume_writing(self) -> None:
assert self._paused
self._paused = False
waiter = self._drain_waiter
if waiter is not None:
self._drain_waiter = None
if not waiter.done():
waiter.set_result(None)
def pause_reading(self) -> None:
self._reading_paused = True
# Parser shouldn't be paused on websockets.
if not self._upgraded:
assert self._parser is not None
self._parser.pause_reading()
if self.transport is not None:
try:
self.transport.pause_reading()
except PAUSE_RESUME_READING_ERRORS:
# Transport lacks flow control; nothing to pause. Intentionally
# ignored (see PAUSE_RESUME_READING_ERRORS; do not use suppress).
pass
def _reading_paused_for_msg_queue(self) -> bool:
"""Keep the transport paused for protocol-specific reasons (overridden)."""
return False
def resume_reading(self, resume_parser: bool = True) -> None:
self._reading_paused = False
# This will resume parsing any unprocessed data from the last pause.
if not self._upgraded and resume_parser:
self.data_received(b"")
# Reading may have been paused again in the above call if there was a lot of
# compressed data still pending.
if (
not self._reading_paused
and not self._reading_paused_for_msg_queue()
and self.transport is not None
):
try:
self.transport.resume_reading()
except PAUSE_RESUME_READING_ERRORS:
# Transport lacks flow control; nothing to resume. Intentionally
# ignored (see PAUSE_RESUME_READING_ERRORS; do not use suppress).
pass
self._reading_paused = False
def connection_made(self, transport: asyncio.BaseTransport) -> None:
tr = cast(asyncio.Transport, transport)
tcp_nodelay(tr, True)
self.transport = tr
def connection_lost(self, exc: BaseException | None) -> None:
# Wake up the writer if currently paused.
self.transport = None
if not self._paused:
return
waiter = self._drain_waiter
if waiter is None:
return
self._drain_waiter = None
if waiter.done():
return
if exc is None:
waiter.set_result(None)
else:
set_exception(
waiter,
ConnectionError("Connection lost"),
exc,
)
async def _drain_helper(self) -> None:
if self.transport is None:
raise ClientConnectionResetError("Connection lost")
if not self._paused:
return
waiter = self._drain_waiter
if waiter is None:
waiter = self._loop.create_future()
self._drain_waiter = waiter
await waiter
File diff suppressed because it is too large Load Diff
+426
View File
@@ -0,0 +1,426 @@
"""HTTP related errors."""
import asyncio
import warnings
from typing import TYPE_CHECKING, Union
from multidict import MultiMapping
from .typedefs import StrOrURL
if TYPE_CHECKING:
import ssl
SSLContext = ssl.SSLContext
else:
try:
import ssl
SSLContext = ssl.SSLContext
except ImportError: # pragma: no cover
ssl = SSLContext = None # type: ignore[assignment]
if TYPE_CHECKING:
from .client_reqrep import ClientResponse, ConnectionKey, Fingerprint, RequestInfo
from .http_parser import RawResponseMessage
else:
RequestInfo = ClientResponse = ConnectionKey = RawResponseMessage = None
__all__ = (
"ClientError",
"ClientConnectionError",
"ClientConnectionResetError",
"ClientOSError",
"ClientConnectorError",
"ClientProxyConnectionError",
"ClientSSLError",
"ClientConnectorDNSError",
"ClientConnectorSSLError",
"ClientConnectorCertificateError",
"ConnectionTimeoutError",
"SocketTimeoutError",
"ServerConnectionError",
"ServerTimeoutError",
"ServerDisconnectedError",
"ServerFingerprintMismatch",
"ClientResponseError",
"ClientHttpProxyError",
"WSServerHandshakeError",
"ContentTypeError",
"ClientPayloadError",
"InvalidURL",
"InvalidUrlClientError",
"RedirectClientError",
"NonHttpUrlClientError",
"InvalidUrlRedirectClientError",
"NonHttpUrlRedirectClientError",
"WSMessageTypeError",
)
class ClientError(Exception):
"""Base class for client connection errors."""
class ClientResponseError(ClientError):
"""Base class for exceptions that occur after getting a response.
request_info: An instance of RequestInfo.
history: A sequence of responses, if redirects occurred.
status: HTTP status code.
message: Error message.
headers: Response headers.
"""
def __init__(
self,
request_info: RequestInfo,
history: tuple[ClientResponse, ...],
*,
code: int | None = None,
status: int | None = None,
message: str = "",
headers: MultiMapping[str] | None = None,
) -> None:
self.request_info = request_info
if code is not None:
if status is not None:
raise ValueError(
"Both code and status arguments are provided; "
"code is deprecated, use status instead"
)
warnings.warn(
"code argument is deprecated, use status instead",
DeprecationWarning,
stacklevel=2,
)
if status is not None:
self.status = status
elif code is not None:
self.status = code
else:
self.status = 0
self.message = message
self.headers = headers
self.history = history
self.args = (request_info, history)
def __str__(self) -> str:
return f"{self.status}, message={self.message!r}, url={str(self.request_info.real_url)!r}"
def __repr__(self) -> str:
args = f"{self.request_info!r}, {self.history!r}"
if self.status != 0:
args += f", status={self.status!r}"
if self.message != "":
args += f", message={self.message!r}"
if self.headers is not None:
args += f", headers={self.headers!r}"
return f"{type(self).__name__}({args})"
@property
def code(self) -> int:
warnings.warn(
"code property is deprecated, use status instead",
DeprecationWarning,
stacklevel=2,
)
return self.status
@code.setter
def code(self, value: int) -> None:
warnings.warn(
"code property is deprecated, use status instead",
DeprecationWarning,
stacklevel=2,
)
self.status = value
class ContentTypeError(ClientResponseError):
"""ContentType found is not valid."""
class WSServerHandshakeError(ClientResponseError):
"""websocket server handshake error."""
class ClientHttpProxyError(ClientResponseError):
"""HTTP proxy error.
Raised in :class:`aiohttp.connector.TCPConnector` if
proxy responds with status other than ``200 OK``
on ``CONNECT`` request.
"""
class TooManyRedirects(ClientResponseError):
"""Client was redirected too many times."""
class ClientConnectionError(ClientError):
"""Base class for client socket errors."""
class ClientConnectionResetError(ClientConnectionError, ConnectionResetError):
"""ConnectionResetError"""
class ClientOSError(ClientConnectionError, OSError):
"""OSError error."""
class ClientConnectorError(ClientOSError):
"""Client connector error.
Raised in :class:`aiohttp.connector.TCPConnector` if
a connection can not be established.
"""
def __init__(self, connection_key: ConnectionKey, os_error: OSError) -> None:
self._conn_key = connection_key
self._os_error = os_error
super().__init__(os_error.errno, os_error.strerror)
self.args = (connection_key, os_error)
@property
def os_error(self) -> OSError:
return self._os_error
@property
def host(self) -> str:
return self._conn_key.host
@property
def port(self) -> int | None:
return self._conn_key.port
@property
def ssl(self) -> Union[SSLContext, bool, "Fingerprint"]:
return self._conn_key.ssl
def __str__(self) -> str:
return "Cannot connect to host {0.host}:{0.port} ssl:{1} [{2}]".format(
self, "default" if self.ssl is True else self.ssl, self.strerror
)
# OSError.__reduce__ does too much black magick
__reduce__ = BaseException.__reduce__
class ClientConnectorDNSError(ClientConnectorError):
"""DNS resolution failed during client connection.
Raised in :class:`aiohttp.connector.TCPConnector` if
DNS resolution fails.
"""
class ClientProxyConnectionError(ClientConnectorError):
"""Proxy connection error.
Raised in :class:`aiohttp.connector.TCPConnector` if
connection to proxy can not be established.
"""
class UnixClientConnectorError(ClientConnectorError):
"""Unix connector error.
Raised in :py:class:`aiohttp.connector.UnixConnector`
if connection to unix socket can not be established.
"""
def __init__(
self, path: str, connection_key: ConnectionKey, os_error: OSError
) -> None:
self._path = path
super().__init__(connection_key, os_error)
@property
def path(self) -> str:
return self._path
def __str__(self) -> str:
return "Cannot connect to unix socket {0.path} ssl:{1} [{2}]".format(
self, "default" if self.ssl is True else self.ssl, self.strerror
)
class ServerConnectionError(ClientConnectionError):
"""Server connection errors."""
class ServerDisconnectedError(ServerConnectionError):
"""Server disconnected."""
def __init__(self, message: RawResponseMessage | str | None = None) -> None:
if message is None:
message = "Server disconnected"
self.args = (message,)
self.message = message
class ServerTimeoutError(ServerConnectionError, asyncio.TimeoutError):
"""Server timeout error."""
class ConnectionTimeoutError(ServerTimeoutError):
"""Connection timeout error."""
class SocketTimeoutError(ServerTimeoutError):
"""Socket timeout error."""
class ServerFingerprintMismatch(ServerConnectionError):
"""SSL certificate does not match expected fingerprint."""
def __init__(self, expected: bytes, got: bytes, host: str, port: int) -> None:
self.expected = expected
self.got = got
self.host = host
self.port = port
self.args = (expected, got, host, port)
def __repr__(self) -> str:
return f"<{self.__class__.__name__} expected={self.expected!r} got={self.got!r} host={self.host!r} port={self.port!r}>"
class ClientPayloadError(ClientError):
"""Response payload error."""
class InvalidURL(ClientError, ValueError):
"""Invalid URL.
URL used for fetching is malformed, e.g. it doesn't contains host
part.
"""
# Derive from ValueError for backward compatibility
def __init__(self, url: StrOrURL, description: str | None = None) -> None:
# The type of url is not yarl.URL because the exception can be raised
# on URL(url) call
self._url = url
self._description = description
if description:
super().__init__(url, description)
else:
super().__init__(url)
@property
def url(self) -> StrOrURL:
return self._url
@property
def description(self) -> "str | None":
return self._description
def __repr__(self) -> str:
return f"<{self.__class__.__name__} {self}>"
def __str__(self) -> str:
if self._description:
return f"{self._url} - {self._description}"
return str(self._url)
class InvalidUrlClientError(InvalidURL):
"""Invalid URL client error."""
class RedirectClientError(ClientError):
"""Client redirect error."""
class NonHttpUrlClientError(ClientError):
"""Non http URL client error."""
class InvalidUrlRedirectClientError(InvalidUrlClientError, RedirectClientError):
"""Invalid URL redirect client error."""
class NonHttpUrlRedirectClientError(NonHttpUrlClientError, RedirectClientError):
"""Non http URL redirect client error."""
class ClientSSLError(ClientConnectorError):
"""Base error for ssl.*Errors."""
if ssl is not None:
cert_errors = (ssl.CertificateError,)
cert_errors_bases = (
ClientSSLError,
ssl.CertificateError,
)
ssl_errors = (ssl.SSLError,)
ssl_error_bases = (ClientSSLError, ssl.SSLError)
else: # pragma: no cover
cert_errors = tuple()
cert_errors_bases = (
ClientSSLError,
ValueError,
)
ssl_errors = tuple()
ssl_error_bases = (ClientSSLError,)
class ClientConnectorSSLError(*ssl_error_bases): # type: ignore[misc]
"""Response ssl error."""
class ClientConnectorCertificateError(*cert_errors_bases): # type: ignore[misc]
"""Response certificate error."""
_conn_key: ConnectionKey
def __init__(
# TODO: If we require ssl in future, this can become ssl.CertificateError
self,
connection_key: ConnectionKey,
certificate_error: Exception,
) -> None:
if isinstance(certificate_error, cert_errors + (OSError,)):
# ssl.CertificateError has errno and strerror, so we should be fine
os_error = certificate_error
else:
os_error = OSError()
super().__init__(connection_key, os_error)
self._certificate_error = certificate_error
self.args = (connection_key, certificate_error)
@property
def certificate_error(self) -> Exception:
return self._certificate_error
@property
def host(self) -> str:
return self._conn_key.host
@property
def port(self) -> int | None:
return self._conn_key.port
@property
def ssl(self) -> bool:
return self._conn_key.is_ssl
def __str__(self) -> str:
return (
f"Cannot connect to host {self.host}:{self.port} ssl:{self.ssl} "
f"[{self.certificate_error.__class__.__name__}: "
f"{self.certificate_error.args}]"
)
class WSMessageTypeError(TypeError):
"""WebSocket message type is not valid."""
@@ -0,0 +1,507 @@
"""
Digest authentication middleware for aiohttp client.
This middleware implements HTTP Digest Authentication according to RFC 7616,
providing a more secure alternative to Basic Authentication. It supports all
standard hash algorithms including MD5, SHA, SHA-256, SHA-512 and their session
variants, as well as both 'auth' and 'auth-int' quality of protection (qop) options.
"""
import hashlib
import os
import re
import sys
import time
from collections.abc import Callable
from typing import Final, Literal, TypedDict
from yarl import URL
from . import hdrs
from .client_exceptions import ClientError
from .client_middlewares import ClientHandlerType
from .client_reqrep import ClientRequest, ClientResponse
from .payload import Payload
class DigestAuthChallenge(TypedDict, total=False):
realm: str
nonce: str
qop: str
algorithm: str
opaque: str
domain: str
stale: str
DigestFunctions: dict[str, Callable[[bytes], "hashlib._Hash"]] = {
"MD5": hashlib.md5,
"MD5-SESS": hashlib.md5,
"SHA": hashlib.sha1,
"SHA-SESS": hashlib.sha1,
"SHA256": hashlib.sha256,
"SHA256-SESS": hashlib.sha256,
"SHA-256": hashlib.sha256,
"SHA-256-SESS": hashlib.sha256,
"SHA512": hashlib.sha512,
"SHA512-SESS": hashlib.sha512,
"SHA-512": hashlib.sha512,
"SHA-512-SESS": hashlib.sha512,
}
# Compile the regex pattern once at module level for performance
_HEADER_PAIRS_PATTERN = re.compile(
r'(?:^|\s|,\s*)(\w+)(?:\s*=\s*(?:"((?:[^"\\]|\\.)*)"|([^\s,]+)))?'
if sys.version_info < (3, 11)
else r'(?:^|\s|,\s*)((?>\w+))(?:\s*=\s*(?:"((?:[^"\\]|\\.)*)"|([^\s,]+)))?'
# +------------|--------|--|--||--|--|----|------|---|---||-----|-> Match valid start/sep
# +--------|--|--||--|--|----|------|---|---||-----|-> alphanumeric key (atomic group reduces backtracking)
# +--|--||--|--|----|------|---|---||-----|-> optional value; absent => bare auth-scheme token
# +--||--|--|----|------|---|---||-----|-> maybe whitespace
# +|--|--|----|------|---|---||-----|-> = (delimiter)
# +--|--|----|------|---|---||-----|-> maybe whitespace
# +--|----|------|---|---||-----|-> group quoted or unquoted
# +----|------|---|---||-----|-> if quoted...
# +------|---|---||-----|-> anything but " or \
# +---|---||-----|-> escaped characters allowed
# +---||-----|-> or can be empty string
# +|-----|-> if unquoted...
# +-----|-> anything but , or <space>
# +-> at least one char req'd
)
# RFC 7616: Challenge parameters to extract
CHALLENGE_FIELDS: Final[
tuple[
Literal["realm", "nonce", "qop", "algorithm", "opaque", "domain", "stale"], ...
]
] = (
"realm",
"nonce",
"qop",
"algorithm",
"opaque",
"domain",
"stale",
)
# Supported digest authentication algorithms
# Use a tuple of sorted keys for predictable documentation and error messages
SUPPORTED_ALGORITHMS: Final[tuple[str, ...]] = tuple(sorted(DigestFunctions.keys()))
# RFC 7616: Fields that require quoting in the Digest auth header
# These fields must be enclosed in double quotes in the Authorization header.
# Algorithm, qop, and nc are never quoted per RFC specifications.
# This frozen set is used by the template-based header construction to
# automatically determine which fields need quotes.
QUOTED_AUTH_FIELDS: Final[frozenset[str]] = frozenset(
{"username", "realm", "nonce", "uri", "response", "opaque", "cnonce"}
)
def escape_quotes(value: str) -> str:
"""Escape backslashes and double quotes for HTTP quoted-strings."""
return value.replace("\\", "\\\\").replace('"', '\\"')
def unescape_quotes(value: str) -> str:
"""Unescape backslashes and double quotes in HTTP quoted-strings."""
return value.replace('\\"', '"').replace("\\\\", "\\")
def parse_header_pairs(header: str) -> dict[str, str]:
"""
Parse key-value pairs from the first challenge of a WWW-Authenticate header.
This function handles the complex format of WWW-Authenticate header values,
supporting both quoted and unquoted values, proper handling of commas in
quoted values, and whitespace variations per RFC 7616.
A single header may carry several challenges
(https://www.rfc-editor.org/rfc/rfc7235#section-4.1). Parsing
stops at the next auth-scheme token so a later challenge's parameters cannot
overwrite the first challenge's values; a leading scheme token is skipped.
Examples of supported formats:
- key1="value1", key2=value2
- key1 = "value1" , key2="value, with, commas"
- key1=value1,key2="value2"
- realm="example.com", nonce="12345", qop="auth"
Args:
header: The header value string to parse
Returns:
Dictionary mapping parameter names to their values
"""
pairs: dict[str, str] = {}
for match in _HEADER_PAIRS_PATTERN.finditer(header):
key = match.group(1)
quoted_val, unquoted_val = match.group(2), match.group(3)
if quoted_val is None and unquoted_val is None:
# Bare token with no "=value": an auth-scheme name, not a parameter.
# Skip a leading scheme; once parameters exist, a new scheme marks
# the start of the next challenge, so stop here.
if pairs:
break
continue
pairs[key] = (
unescape_quotes(quoted_val) if quoted_val is not None else unquoted_val
)
return pairs
class DigestAuthMiddleware:
"""
HTTP digest authentication middleware for aiohttp client.
This middleware intercepts 401 Unauthorized responses containing a Digest
authentication challenge, calculates the appropriate digest credentials,
and automatically retries the request with the proper Authorization header.
Features:
- Handles all aspects of Digest authentication handshake automatically
- Supports all standard hash algorithms:
- MD5, MD5-SESS
- SHA, SHA-SESS
- SHA256, SHA256-SESS, SHA-256, SHA-256-SESS
- SHA512, SHA512-SESS, SHA-512, SHA-512-SESS
- Supports 'auth' and 'auth-int' quality of protection modes
- Properly handles quoted strings and parameter parsing
- Includes replay attack protection with client nonce count tracking
- Supports preemptive authentication per RFC 7616 Section 3.6
Origin scoping:
The credentials are scoped to the origin of the first request the
middleware handles. A request to a different origin is passed through
untouched, so it never receives a digest response computed from those
credentials, unless that origin falls within a protection space the
anchor origin advertised through the RFC 7616 ``domain`` directive. Make
the first request through the middleware against the intended origin, as
the anchor is pinned to it and not reset for the life of the instance.
Standards compliance:
- RFC 7616: HTTP Digest Access Authentication (primary reference)
- RFC 2617: HTTP Authentication (deprecated by RFC 7616)
- RFC 1945: Section 11.1 (username restrictions)
Implementation notes:
The core digest calculation is inspired by the implementation in
https://github.com/requests/requests/blob/v2.18.4/requests/auth.py
with added support for modern digest auth features and error handling.
"""
def __init__(
self,
login: str,
password: str,
preemptive: bool = True,
) -> None:
if login is None:
raise ValueError("None is not allowed as login value")
if password is None:
raise ValueError("None is not allowed as password value")
if ":" in login:
raise ValueError('A ":" is not allowed in username (RFC 1945#section-11.1)')
self._login_str: Final[str] = login
self._login_bytes: Final[bytes] = login.encode("utf-8")
self._password_bytes: Final[bytes] = password.encode("utf-8")
self._last_nonce_bytes = b""
self._nonce_count = 0
self._challenge: DigestAuthChallenge = {}
self._preemptive: bool = preemptive
# Set of URLs defining the protection space
self._protection_space: list[str] = []
# Origin the credentials are scoped to; set on the first request.
self._origin: URL | None = None
async def _encode(self, method: str, url: URL, body: Payload | Literal[b""]) -> str:
"""
Build digest authorization header for the current challenge.
Args:
method: The HTTP method (GET, POST, etc.)
url: The request URL
body: The request body (used for qop=auth-int)
Returns:
A fully formatted Digest authorization header string
Raises:
ClientError: If the challenge is missing required parameters or
contains unsupported values
"""
challenge = self._challenge
if "realm" not in challenge:
raise ClientError(
"Malformed Digest auth challenge: Missing 'realm' parameter"
)
if "nonce" not in challenge:
raise ClientError(
"Malformed Digest auth challenge: Missing 'nonce' parameter"
)
# Empty realm values are allowed per RFC 7616 (SHOULD, not MUST, contain host name)
realm = challenge["realm"]
nonce = challenge["nonce"]
# Empty nonce values are not allowed as they are security-critical for replay protection
if not nonce:
raise ClientError(
"Security issue: Digest auth challenge contains empty 'nonce' value"
)
qop_raw = challenge.get("qop", "")
# Preserve original algorithm case for response while using uppercase for processing
algorithm_original = challenge.get("algorithm", "MD5")
algorithm = algorithm_original.upper()
opaque = challenge.get("opaque", "")
# Convert string values to bytes once
nonce_bytes = nonce.encode("utf-8")
realm_bytes = realm.encode("utf-8")
# Use the encoded request-target (raw_path_qs) since that is what is
# transmitted on the wire and what the server signs against. Using the
# decoded form would cause digest verification to fail when the path
# or query string contains percent-encoded reserved characters.
path = URL(url).raw_path_qs
# Process QoP
qop = ""
qop_bytes = b""
if qop_raw:
valid_qops = {"auth", "auth-int"}.intersection(
{q.strip() for q in qop_raw.split(",") if q.strip()}
)
if not valid_qops:
raise ClientError(
f"Digest auth error: Unsupported Quality of Protection (qop) value(s): {qop_raw}"
)
qop = "auth-int" if "auth-int" in valid_qops else "auth"
qop_bytes = qop.encode("utf-8")
if algorithm not in DigestFunctions:
raise ClientError(
f"Digest auth error: Unsupported hash algorithm: {algorithm}. "
f"Supported algorithms: {', '.join(SUPPORTED_ALGORITHMS)}"
)
hash_fn: Final = DigestFunctions[algorithm]
def H(x: bytes) -> bytes:
"""RFC 7616 Section 3: Hash function H(data) = hex(hash(data))."""
return hash_fn(x).hexdigest().encode()
def KD(s: bytes, d: bytes) -> bytes:
"""RFC 7616 Section 3: KD(secret, data) = H(concat(secret, ":", data))."""
return H(b":".join((s, d)))
# Calculate A1 and A2
A1 = b":".join((self._login_bytes, realm_bytes, self._password_bytes))
A2 = f"{method.upper()}:{path}".encode()
if qop == "auth-int":
if isinstance(body, Payload): # will always be empty bytes unless Payload
entity_bytes = await body.as_bytes() # Get bytes from Payload
else:
entity_bytes = body
entity_hash = H(entity_bytes)
A2 = b":".join((A2, entity_hash))
HA1 = H(A1)
HA2 = H(A2)
# Nonce count handling
if nonce_bytes == self._last_nonce_bytes:
self._nonce_count += 1
else:
self._nonce_count = 1
self._last_nonce_bytes = nonce_bytes
ncvalue = f"{self._nonce_count:08x}"
ncvalue_bytes = ncvalue.encode("utf-8")
# Generate client nonce
cnonce = hashlib.sha1(
b"".join(
[
str(self._nonce_count).encode("utf-8"),
nonce_bytes,
time.ctime().encode("utf-8"),
os.urandom(8),
]
)
).hexdigest()[:16]
cnonce_bytes = cnonce.encode("utf-8")
# Special handling for session-based algorithms
if algorithm.upper().endswith("-SESS"):
HA1 = H(b":".join((HA1, nonce_bytes, cnonce_bytes)))
# Calculate the response digest
if qop:
noncebit = b":".join(
(nonce_bytes, ncvalue_bytes, cnonce_bytes, qop_bytes, HA2)
)
response_digest = KD(HA1, noncebit)
else:
response_digest = KD(HA1, b":".join((nonce_bytes, HA2)))
# Define a dict mapping of header fields to their values
# Group fields into always-present, optional, and qop-dependent
header_fields = {
# Always present fields
"username": escape_quotes(self._login_str),
"realm": escape_quotes(realm),
"nonce": escape_quotes(nonce),
"uri": path,
"response": response_digest.decode(),
"algorithm": algorithm_original,
}
# Optional fields
if opaque:
header_fields["opaque"] = escape_quotes(opaque)
# QoP-dependent fields
if qop:
header_fields["qop"] = qop
header_fields["nc"] = ncvalue
header_fields["cnonce"] = cnonce
# Build header using templates for each field type
pairs: list[str] = []
for field, value in header_fields.items():
if field in QUOTED_AUTH_FIELDS:
pairs.append(f'{field}="{value}"')
else:
pairs.append(f"{field}={value}")
return f"Digest {', '.join(pairs)}"
def _in_protection_space(self, url: URL) -> bool:
"""
Check if the given URL is within the current protection space.
According to RFC 7616, a URI is in the protection space if any URI
in the protection space is a prefix of it (after both have been made absolute).
"""
request_str = str(url)
for space_str in self._protection_space:
# Check if request starts with space URL
if not request_str.startswith(space_str):
continue
# Exact match or space ends with / (proper directory prefix)
if len(request_str) == len(space_str) or space_str[-1] == "/":
return True
# Check next char is / to ensure proper path boundary
if request_str[len(space_str)] == "/":
return True
return False
def _authenticate(self, response: ClientResponse) -> bool:
"""
Takes the given response and tries digest-auth, if needed.
Returns true if the original request must be resent.
"""
if response.status != 401:
return False
auth_header = response.headers.get("www-authenticate", "")
if not auth_header:
return False # No authentication header present
method, sep, headers = auth_header.partition(" ")
if not sep:
# No space found in www-authenticate header
return False # Malformed auth header, missing scheme separator
if method.lower() != "digest":
# Not a digest auth challenge (could be Basic, Bearer, etc.)
return False
if not headers:
# We have a digest scheme but no parameters
return False # Malformed digest header, missing parameters
# We have a digest auth header with content
if not (header_pairs := parse_header_pairs(headers)):
# Failed to parse any key-value pairs
return False # Malformed digest header, no valid parameters
# Extract challenge parameters
self._challenge = {}
for field in CHALLENGE_FIELDS:
if (value := header_pairs.get(field)) is not None:
self._challenge[field] = value
# Update protection space based on domain parameter or default to origin
origin = response.url.origin()
self._protection_space = []
if domain := self._challenge.get("domain"):
# Parse space-separated list of URIs
for uri in domain.split():
# Remove quotes if present
uri = uri.strip('"')
if not uri:
continue
if uri.startswith("/"):
# Path-absolute, relative to origin
self._protection_space.append(str(origin.join(URL(uri))))
else:
# Absolute URI
self._protection_space.append(str(URL(uri)))
if not self._protection_space:
self._protection_space = [str(origin)]
# Return True only if we found at least one challenge parameter
return bool(self._challenge)
async def __call__(
self, request: ClientRequest, handler: ClientHandlerType
) -> ClientResponse:
"""Run the digest auth middleware."""
# Credentials are scoped to the first request's origin. Other origins
# pass through untouched unless a challenge from the anchor origin
# advertised them via RFC 7616 domain; mirrors aiohttp stripping
# Authorization on cross-origin redirects.
origin = request.url.origin()
if self._origin is None:
self._origin = origin
elif origin != self._origin and not self._in_protection_space(request.url):
return await handler(request)
response = None
for retry_count in range(2):
# Apply authorization header if:
# 1. This is a retry after 401 (retry_count > 0), OR
# 2. Preemptive auth is enabled AND we have a challenge AND the URL is in protection space
if retry_count > 0 or (
self._preemptive
and self._challenge
and self._in_protection_space(request.url)
):
request.headers[hdrs.AUTHORIZATION] = await self._encode(
request.method, request.url, request.body
)
# Send the request
response = await handler(request)
# Check if we need to authenticate
if not self._authenticate(response):
break
# At this point, response is guaranteed to be defined
assert response is not None
return response
+55
View File
@@ -0,0 +1,55 @@
"""Client middleware support."""
from collections.abc import Awaitable, Callable, Sequence
from .client_reqrep import ClientRequest, ClientResponse
__all__ = ("ClientMiddlewareType", "ClientHandlerType", "build_client_middlewares")
# Type alias for client request handlers - functions that process requests and return responses
ClientHandlerType = Callable[[ClientRequest], Awaitable[ClientResponse]]
# Type for client middleware - similar to server but uses ClientRequest/ClientResponse
ClientMiddlewareType = Callable[
[ClientRequest, ClientHandlerType], Awaitable[ClientResponse]
]
def build_client_middlewares(
handler: ClientHandlerType,
middlewares: Sequence[ClientMiddlewareType],
) -> ClientHandlerType:
"""
Apply middlewares to request handler.
The middlewares are applied in reverse order, so the first middleware
in the list wraps all subsequent middlewares and the handler.
This implementation avoids using partial/update_wrapper to minimize overhead
and doesn't cache to avoid holding references to stateful middleware.
"""
# Optimize for single middleware case
if len(middlewares) == 1:
middleware = middlewares[0]
async def single_middleware_handler(req: ClientRequest) -> ClientResponse:
return await middleware(req, handler)
return single_middleware_handler
# Build the chain for multiple middlewares
current_handler = handler
for middleware in reversed(middlewares):
# Create a new closure that captures the current state
def make_wrapper(
mw: ClientMiddlewareType, next_h: ClientHandlerType
) -> ClientHandlerType:
async def wrapped(req: ClientRequest) -> ClientResponse:
return await mw(req, next_h)
return wrapped
current_handler = make_wrapper(middleware, current_handler)
return current_handler
+372
View File
@@ -0,0 +1,372 @@
import asyncio
from contextlib import suppress
from typing import Any, Callable
from .base_protocol import BaseProtocol
from .client_exceptions import (
ClientConnectionError,
ClientOSError,
ClientPayloadError,
ServerDisconnectedError,
SocketTimeoutError,
)
from .helpers import (
_EXC_SENTINEL,
DEFAULT_CHUNK_SIZE,
EMPTY_BODY_STATUS_CODES,
BaseTimerContext,
set_exception,
set_result,
)
from .http import HttpResponseParser, RawResponseMessage
from .http_exceptions import HttpProcessingError
from .streams import EMPTY_PAYLOAD, DataQueue, StreamReader
class ResponseHandler(BaseProtocol, DataQueue[tuple[RawResponseMessage, StreamReader]]):
"""Helper class to adapt between Protocol and StreamReader."""
def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
BaseProtocol.__init__(self, loop=loop, parser=None)
DataQueue.__init__(self, loop)
self._should_close = False
self._payload: StreamReader | None = None
self._skip_payload = False
self._payload_parser = None
self._data_received_cb: Callable[[], None] | None = None
self._timer = None
self._tail = b""
self._read_timeout: float | None = None
self._read_timeout_handle: asyncio.TimerHandle | None = None
self._timeout_ceil_threshold: float | None = 5
self._closed: None | asyncio.Future[None] = None
self._connection_lost_called = False
@property
def closed(self) -> None | asyncio.Future[None]:
"""Future that is set when the connection is closed.
This property returns a Future that will be completed when the connection
is closed. The Future is created lazily on first access to avoid creating
futures that will never be awaited.
Returns:
- A Future[None] if the connection is still open or was closed after
this property was accessed
- None if connection_lost() was already called before this property
was ever accessed (indicating no one is waiting for the closure)
"""
if self._closed is None and not self._connection_lost_called:
self._closed = self._loop.create_future()
return self._closed
@property
def upgraded(self) -> bool:
return self._upgraded
@property
def should_close(self) -> bool:
return bool(
self._should_close
or (self._payload is not None and not self._payload.is_eof())
or self._upgraded
or self._exception is not None
or self._payload_parser is not None
or self._buffer
or self._tail
)
def force_close(self) -> None:
self._should_close = True
def close(self) -> None:
self._exception = None # Break cyclic references
transport = self.transport
if transport is not None:
transport.close()
self.transport = None
self._payload = None
self._drop_timeout()
def abort(self) -> None:
self._exception = None # Break cyclic references
transport = self.transport
if transport is not None:
transport.abort()
self.transport = None
self._payload = None
self._drop_timeout()
def is_connected(self) -> bool:
return self.transport is not None and not self.transport.is_closing()
def connection_lost(self, exc: BaseException | None) -> None:
self._connection_lost_called = True
self._drop_timeout()
original_connection_error = exc
reraised_exc = original_connection_error
connection_closed_cleanly = original_connection_error is None
if self._closed is not None:
# If someone is waiting for the closed future,
# we should set it to None or an exception. If
# self._closed is None, it means that
# connection_lost() was called already
# or nobody is waiting for it.
if connection_closed_cleanly:
set_result(self._closed, None)
else:
assert original_connection_error is not None
set_exception(
self._closed,
ClientConnectionError(
f"Connection lost: {original_connection_error !s}",
),
original_connection_error,
)
if self._payload_parser is not None:
with suppress(Exception): # FIXME: log this somehow?
self._payload_parser.feed_eof()
uncompleted = None
if self._parser is not None:
try:
uncompleted = self._parser.feed_eof()
except Exception as underlying_exc:
if self._payload is not None:
client_payload_exc_msg = (
f"Response payload is not completed: {underlying_exc !r}"
)
if not connection_closed_cleanly:
client_payload_exc_msg = (
f"{client_payload_exc_msg !s}. "
f"{original_connection_error !r}"
)
set_exception(
self._payload,
ClientPayloadError(client_payload_exc_msg),
underlying_exc,
)
if not self.is_eof():
if isinstance(original_connection_error, OSError):
reraised_exc = ClientOSError(*original_connection_error.args)
if connection_closed_cleanly:
reraised_exc = ServerDisconnectedError(uncompleted)
# assigns self._should_close to True as side effect,
# we do it anyway below
underlying_non_eof_exc = (
_EXC_SENTINEL
if connection_closed_cleanly
else original_connection_error
)
assert underlying_non_eof_exc is not None
assert reraised_exc is not None
self.set_exception(reraised_exc, underlying_non_eof_exc)
self._should_close = True
self._parser = None
self._payload = None
self._payload_parser = None
self._reading_paused = False
super().connection_lost(reraised_exc)
def eof_received(self) -> None:
# should call parser.feed_eof() most likely
self._drop_timeout()
def pause_reading(self) -> None:
super().pause_reading()
self._drop_timeout()
def resume_reading(self, resume_parser: bool = True) -> None:
was_paused = self._reading_paused
super().resume_reading(resume_parser)
if was_paused:
self._reschedule_timeout()
def set_exception(
self,
exc: BaseException,
exc_cause: BaseException = _EXC_SENTINEL,
) -> None:
self._should_close = True
self._drop_timeout()
super().set_exception(exc, exc_cause)
def set_parser(
self,
parser: Any,
payload: Any,
data_received_cb: Callable[[], None] | None = None,
) -> None:
# TODO: actual types are:
# parser: WebSocketReader
# payload: WebSocketDataQueue
# but they are not generi enough
# Need an ABC for both types
self._payload = payload
self._payload_parser = parser
self._data_received_cb = data_received_cb
self._drop_timeout()
if self._tail:
data, self._tail = self._tail, b""
self.data_received(data)
def set_response_params(
self,
*,
timer: BaseTimerContext | None = None,
skip_payload: bool = False,
read_until_eof: bool = False,
auto_decompress: bool = True,
read_timeout: float | None = None,
read_bufsize: int = DEFAULT_CHUNK_SIZE,
timeout_ceil_threshold: float = 5,
max_line_size: int = 8190,
max_field_size: int = 8190,
max_headers: int = 128,
) -> None:
self._skip_payload = skip_payload
self._read_timeout = read_timeout
self._timeout_ceil_threshold = timeout_ceil_threshold
self._parser = HttpResponseParser(
self,
self._loop,
read_bufsize,
timer=timer,
payload_exception=ClientPayloadError,
response_with_body=not skip_payload,
read_until_eof=read_until_eof,
auto_decompress=auto_decompress,
max_line_size=max_line_size,
max_field_size=max_field_size,
max_headers=max_headers,
)
if self._tail:
data, self._tail = self._tail, b""
self.data_received(data)
def _drop_timeout(self) -> None:
if self._read_timeout_handle is not None:
self._read_timeout_handle.cancel()
self._read_timeout_handle = None
def _reschedule_timeout(self) -> None:
timeout = self._read_timeout
if self._read_timeout_handle is not None:
self._read_timeout_handle.cancel()
if timeout:
self._read_timeout_handle = self._loop.call_later(
timeout, self._on_read_timeout
)
else:
self._read_timeout_handle = None
def start_timeout(self) -> None:
self._reschedule_timeout()
@property
def read_timeout(self) -> float | None:
return self._read_timeout
@read_timeout.setter
def read_timeout(self, read_timeout: float | None) -> None:
self._read_timeout = read_timeout
def _on_read_timeout(self) -> None:
exc = SocketTimeoutError("Timeout on reading data from socket")
self.set_exception(exc)
if self._payload is not None:
set_exception(self._payload, exc)
def data_received(self, data: bytes) -> None:
# If no data, then we are resuming decompression. We haven't received
# data from the socket, so we can avoid the reschedule overhead.
if data:
self._reschedule_timeout()
# custom payload parser - currently always WebSocketReader
if self._payload_parser is not None:
if self._data_received_cb is not None:
self._data_received_cb()
eof, tail = self._payload_parser.feed_data(data)
if eof:
self._payload = None
self._payload_parser = None
if tail:
self.data_received(tail)
return
if self._upgraded or self._parser is None:
# i.e. websocket connection, websocket parser is not set yet
self._tail += data
return
# parse http messages
try:
messages, upgraded, tail = self._parser.feed_data(data)
except BaseException as underlying_exc:
if self.transport is not None:
# connection.release() could be called BEFORE
# data_received(), the transport is already
# closed in this case
self.transport.close()
if not isinstance(underlying_exc, Exception):
raise
# should_close is True after the call
if isinstance(underlying_exc, HttpProcessingError):
exc = HttpProcessingError(
code=underlying_exc.code,
message=underlying_exc.message,
headers=underlying_exc.headers,
)
else:
exc = HttpProcessingError()
self.set_exception(exc, underlying_exc)
return
self._upgraded = upgraded
payload: StreamReader | None = None
for message, payload in messages:
if message.should_close:
self._should_close = True
self._payload = payload
if self._skip_payload or message.code in EMPTY_BODY_STATUS_CODES:
self.feed_data((message, EMPTY_PAYLOAD), 0)
else:
self.feed_data((message, payload), 0)
if payload is not None:
# new message(s) was processed
# register timeout handler unsubscribing
# either on end-of-stream or immediately for
# EMPTY_PAYLOAD
if payload is not EMPTY_PAYLOAD:
payload.on_eof(self._drop_timeout)
else:
self._drop_timeout()
if upgraded and tail:
self.data_received(tail)
File diff suppressed because it is too large Load Diff
+560
View File
@@ -0,0 +1,560 @@
"""WebSocket client for asyncio."""
import asyncio
import sys
from collections.abc import Callable
from types import TracebackType
from typing import Any, Generic, Literal, Optional, cast, overload
import attr
from ._websocket.reader import WebSocketDataQueue
from .client_exceptions import ClientError, ServerTimeoutError, WSMessageTypeError
from .client_reqrep import ClientResponse
from .helpers import calculate_timeout_when, set_result
from .http import (
WS_CLOSED_MESSAGE,
WS_CLOSING_MESSAGE,
WebSocketError,
WSCloseCode,
WSMessage,
WSMessageDecodeText,
WSMessageNoDecodeText,
WSMsgType,
)
from .http_websocket import _INTERNAL_RECEIVE_TYPES, WebSocketWriter
from .streams import EofStream
from .typedefs import (
DEFAULT_JSON_DECODER,
DEFAULT_JSON_ENCODER,
JSONBytesEncoder,
JSONDecoder,
JSONEncoder,
)
if sys.version_info >= (3, 13):
from typing import TypeVar
else:
from typing_extensions import TypeVar
if sys.version_info >= (3, 11):
import asyncio as async_timeout
from typing import Self
else:
import async_timeout
from typing_extensions import Self
# TypeVar for whether text messages are decoded to str (True) or kept as bytes (False)
# Covariant because it only affects return types, not input types
_DecodeText = TypeVar("_DecodeText", bound=bool, covariant=True, default=Literal[True])
@attr.s(frozen=True, slots=True)
class ClientWSTimeout:
ws_receive = attr.ib(type=Optional[float], default=None)
ws_close = attr.ib(type=Optional[float], default=None)
DEFAULT_WS_CLIENT_TIMEOUT = ClientWSTimeout(ws_receive=None, ws_close=10.0)
class ClientWebSocketResponse(Generic[_DecodeText]):
def __init__(
self,
reader: WebSocketDataQueue,
writer: WebSocketWriter,
protocol: str | None,
response: ClientResponse,
timeout: ClientWSTimeout,
autoclose: bool,
autoping: bool,
loop: asyncio.AbstractEventLoop,
*,
heartbeat: float | None = None,
compress: int = 0,
client_notakeover: bool = False,
) -> None:
self._response = response
self._conn = response.connection
self._writer = writer
self._reader = reader
self._protocol = protocol
self._closed = False
self._closing = False
self._close_code: int | None = None
self._timeout = timeout
self._autoclose = autoclose
self._autoping = autoping
self._heartbeat = heartbeat
self._heartbeat_cb: asyncio.TimerHandle | None = None
self._heartbeat_when: float = 0.0
if heartbeat is not None:
self._pong_heartbeat = heartbeat / 2.0
self._pong_response_cb: asyncio.TimerHandle | None = None
self._loop = loop
self._waiting: bool = False
self._close_wait: asyncio.Future[None] | None = None
self._exception: BaseException | None = None
self._compress = compress
self._client_notakeover = client_notakeover
self._ping_task: asyncio.Task[None] | None = None
self._need_heartbeat_reset = False
self._heartbeat_reset_handle: asyncio.Handle | None = None
self._reset_heartbeat()
def _cancel_heartbeat(self) -> None:
self._cancel_pong_response_cb()
if self._heartbeat_reset_handle is not None:
self._heartbeat_reset_handle.cancel()
self._heartbeat_reset_handle = None
self._need_heartbeat_reset = False
if self._heartbeat_cb is not None:
self._heartbeat_cb.cancel()
self._heartbeat_cb = None
if self._ping_task is not None:
self._ping_task.cancel()
self._ping_task = None
def _cancel_pong_response_cb(self) -> None:
if self._pong_response_cb is not None:
self._pong_response_cb.cancel()
self._pong_response_cb = None
def _on_data_received(self) -> None:
if self._heartbeat is None or self._need_heartbeat_reset:
return
loop = self._loop
assert loop is not None
# Coalesce multiple chunks received in the same loop tick into a single
# heartbeat reset. Resetting immediately per chunk increases timer churn.
self._need_heartbeat_reset = True
self._heartbeat_reset_handle = loop.call_soon(self._flush_heartbeat_reset)
def _flush_heartbeat_reset(self) -> None:
self._heartbeat_reset_handle = None
if not self._need_heartbeat_reset:
return
self._reset_heartbeat()
self._need_heartbeat_reset = False
def _reset_heartbeat(self) -> None:
if self._heartbeat is None:
return
self._cancel_pong_response_cb()
loop = self._loop
assert loop is not None
conn = self._conn
timeout_ceil_threshold = (
conn._connector._timeout_ceil_threshold if conn is not None else 5
)
now = loop.time()
when = calculate_timeout_when(now, self._heartbeat, timeout_ceil_threshold)
self._heartbeat_when = when
if self._heartbeat_cb is None:
# We do not cancel the previous heartbeat_cb here because
# it generates a significant amount of TimerHandle churn
# which causes asyncio to rebuild the heap frequently.
# Instead _send_heartbeat() will reschedule the next
# heartbeat if it fires too early.
self._heartbeat_cb = loop.call_at(when, self._send_heartbeat)
def _send_heartbeat(self) -> None:
self._heartbeat_cb = None
# If heartbeat reset is pending (data is being received), skip sending
# the ping and let the reset callback handle rescheduling the heartbeat.
if self._need_heartbeat_reset:
return
loop = self._loop
now = loop.time()
if now < self._heartbeat_when:
# Heartbeat fired too early, reschedule
self._heartbeat_cb = loop.call_at(
self._heartbeat_when, self._send_heartbeat
)
return
conn = self._conn
timeout_ceil_threshold = (
conn._connector._timeout_ceil_threshold if conn is not None else 5
)
when = calculate_timeout_when(now, self._pong_heartbeat, timeout_ceil_threshold)
self._cancel_pong_response_cb()
self._pong_response_cb = loop.call_at(when, self._pong_not_received)
coro = self._writer.send_frame(b"", WSMsgType.PING)
if sys.version_info >= (3, 12):
# Optimization for Python 3.12, try to send the ping
# immediately to avoid having to schedule
# the task on the event loop.
ping_task = asyncio.Task(coro, loop=loop, eager_start=True)
else:
ping_task = loop.create_task(coro)
if not ping_task.done():
self._ping_task = ping_task
ping_task.add_done_callback(self._ping_task_done)
else:
self._ping_task_done(ping_task)
def _ping_task_done(self, task: "asyncio.Task[None]") -> None:
"""Callback for when the ping task completes."""
if not task.cancelled() and (exc := task.exception()):
self._handle_ping_pong_exception(exc)
self._ping_task = None
def _pong_not_received(self) -> None:
self._handle_ping_pong_exception(
ServerTimeoutError(f"No PONG received after {self._pong_heartbeat} seconds")
)
def _handle_ping_pong_exception(self, exc: BaseException) -> None:
"""Handle exceptions raised during ping/pong processing."""
if self._closed:
return
self._set_closed()
self._close_code = WSCloseCode.ABNORMAL_CLOSURE
self._exception = exc
self._response.close()
if self._waiting and not self._closing:
self._reader.feed_data(WSMessage(WSMsgType.ERROR, exc, None), 0)
def _set_closed(self) -> None:
"""Set the connection to closed.
Cancel any heartbeat timers and set the closed flag.
"""
self._closed = True
self._cancel_heartbeat()
def _set_closing(self) -> None:
"""Set the connection to closing.
Cancel any heartbeat timers and set the closing flag.
"""
self._closing = True
self._cancel_heartbeat()
@property
def closed(self) -> bool:
return self._closed
@property
def close_code(self) -> int | None:
return self._close_code
@property
def protocol(self) -> str | None:
return self._protocol
@property
def compress(self) -> int:
return self._compress
@property
def client_notakeover(self) -> bool:
return self._client_notakeover
def get_extra_info(self, name: str, default: Any = None) -> Any:
"""extra info from connection transport"""
conn = self._response.connection
if conn is None:
return default
transport = conn.transport
if transport is None:
return default
return transport.get_extra_info(name, default)
def exception(self) -> BaseException | None:
return self._exception
async def ping(self, message: bytes = b"") -> None:
await self._writer.send_frame(message, WSMsgType.PING)
async def pong(self, message: bytes = b"") -> None:
await self._writer.send_frame(message, WSMsgType.PONG)
async def send_frame(
self, message: bytes, opcode: WSMsgType, compress: int | None = None
) -> None:
"""Send a frame over the websocket."""
await self._writer.send_frame(message, opcode, compress)
async def send_str(self, data: str, compress: int | None = None) -> None:
if not isinstance(data, str):
raise TypeError("data argument must be str (%r)" % type(data))
await self._writer.send_frame(
data.encode("utf-8"), WSMsgType.TEXT, compress=compress
)
async def send_bytes(self, data: bytes, compress: int | None = None) -> None:
if not isinstance(data, (bytes, bytearray, memoryview)):
raise TypeError("data argument must be byte-ish (%r)" % type(data))
await self._writer.send_frame(data, WSMsgType.BINARY, compress=compress)
async def send_json(
self,
data: Any,
compress: int | None = None,
*,
dumps: JSONEncoder = DEFAULT_JSON_ENCODER,
) -> None:
await self.send_str(dumps(data), compress=compress)
async def send_json_bytes(
self,
data: Any,
compress: int | None = None,
*,
dumps: JSONBytesEncoder,
) -> None:
"""Send JSON data using a bytes-returning encoder as a binary frame.
Use this when your JSON encoder (like orjson) returns bytes
instead of str, avoiding the encode/decode overhead.
"""
await self.send_bytes(dumps(data), compress=compress)
async def close(self, *, code: int = WSCloseCode.OK, message: bytes = b"") -> bool:
# we need to break `receive()` cycle first,
# `close()` may be called from different task
if self._waiting and not self._closing:
assert self._loop is not None
self._close_wait = self._loop.create_future()
self._set_closing()
self._reader.feed_data(WS_CLOSING_MESSAGE, 0)
await self._close_wait
if self._closed:
return False
self._set_closed()
try:
await self._writer.close(code, message)
except asyncio.CancelledError:
self._close_code = WSCloseCode.ABNORMAL_CLOSURE
self._response.close()
raise
except Exception as exc:
self._close_code = WSCloseCode.ABNORMAL_CLOSURE
self._exception = exc
self._response.close()
return True
if self._close_code:
self._response.close()
return True
while True:
try:
async with async_timeout.timeout(self._timeout.ws_close):
msg = await self._reader.read()
except asyncio.CancelledError:
self._close_code = WSCloseCode.ABNORMAL_CLOSURE
self._response.close()
raise
except Exception as exc:
self._close_code = WSCloseCode.ABNORMAL_CLOSURE
self._exception = exc
self._response.close()
return True
if msg.type is WSMsgType.CLOSE:
self._close_code = msg.data
self._response.close()
return True
@overload
async def receive(
self: "ClientWebSocketResponse[Literal[True]]", timeout: float | None = None
) -> WSMessageDecodeText: ...
@overload
async def receive(
self: "ClientWebSocketResponse[Literal[False]]", timeout: float | None = None
) -> WSMessageNoDecodeText: ...
@overload
async def receive(
self: "ClientWebSocketResponse[_DecodeText]", timeout: float | None = None
) -> WSMessageDecodeText | WSMessageNoDecodeText: ...
async def receive(
self, timeout: float | None = None
) -> WSMessageDecodeText | WSMessageNoDecodeText:
receive_timeout = timeout or self._timeout.ws_receive
while True:
if self._waiting:
raise RuntimeError("Concurrent call to receive() is not allowed")
if self._closed:
return WS_CLOSED_MESSAGE
elif self._closing:
await self.close()
return WS_CLOSED_MESSAGE
try:
self._waiting = True
try:
if receive_timeout:
# Entering the context manager and creating
# Timeout() object can take almost 50% of the
# run time in this loop so we avoid it if
# there is no read timeout.
async with async_timeout.timeout(receive_timeout):
msg = await self._reader.read()
else:
msg = await self._reader.read()
finally:
self._waiting = False
if self._close_wait:
set_result(self._close_wait, None)
except (asyncio.CancelledError, asyncio.TimeoutError):
self._close_code = WSCloseCode.ABNORMAL_CLOSURE
raise
except EofStream:
self._close_code = WSCloseCode.OK
await self.close()
return WSMessage(WSMsgType.CLOSED, None, None)
except ClientError:
# Likely ServerDisconnectedError when connection is lost
self._set_closed()
self._close_code = WSCloseCode.ABNORMAL_CLOSURE
return WS_CLOSED_MESSAGE
except WebSocketError as exc:
self._close_code = exc.code
await self.close(code=exc.code)
return WSMessage(WSMsgType.ERROR, exc, None)
except Exception as exc:
self._exception = exc
self._set_closing()
self._close_code = WSCloseCode.ABNORMAL_CLOSURE
await self.close()
return WSMessage(WSMsgType.ERROR, exc, None)
if msg.type not in _INTERNAL_RECEIVE_TYPES:
# If its not a close/closing/ping/pong message
# we can return it immediately
return msg
if msg.type is WSMsgType.CLOSE:
self._set_closing()
self._close_code = msg.data
if not self._closed and self._autoclose:
await self.close()
elif msg.type is WSMsgType.CLOSING:
self._set_closing()
elif msg.type is WSMsgType.PING and self._autoping:
await self.pong(msg.data)
continue
elif msg.type is WSMsgType.PONG and self._autoping:
continue
return msg
@overload
async def receive_str(
self: "ClientWebSocketResponse[Literal[True]]", *, timeout: float | None = None
) -> str: ...
@overload
async def receive_str(
self: "ClientWebSocketResponse[Literal[False]]", *, timeout: float | None = None
) -> bytes: ...
@overload
async def receive_str(
self: "ClientWebSocketResponse[_DecodeText]", *, timeout: float | None = None
) -> str | bytes: ...
async def receive_str(self, *, timeout: float | None = None) -> str | bytes:
"""Receive TEXT message.
Returns str when decode_text=True (default), bytes when decode_text=False.
"""
msg = await self.receive(timeout)
if msg.type is not WSMsgType.TEXT:
raise WSMessageTypeError(
f"Received message {msg.type}:{msg.data!r} is not WSMsgType.TEXT"
)
return cast(str, msg.data)
async def receive_bytes(self, *, timeout: float | None = None) -> bytes:
msg = await self.receive(timeout)
if msg.type is not WSMsgType.BINARY:
raise WSMessageTypeError(
f"Received message {msg.type}:{msg.data!r} is not WSMsgType.BINARY"
)
return cast(bytes, msg.data)
@overload
async def receive_json(
self: "ClientWebSocketResponse[Literal[True]]",
*,
loads: JSONDecoder = ...,
timeout: float | None = None,
) -> Any: ...
@overload
async def receive_json(
self: "ClientWebSocketResponse[Literal[False]]",
*,
loads: Callable[[bytes], Any] = ...,
timeout: float | None = None,
) -> Any: ...
@overload
async def receive_json(
self: "ClientWebSocketResponse[_DecodeText]",
*,
loads: JSONDecoder | Callable[[bytes], Any] = ...,
timeout: float | None = None,
) -> Any: ...
async def receive_json(
self,
*,
loads: JSONDecoder | Callable[[bytes], Any] = DEFAULT_JSON_DECODER,
timeout: float | None = None,
) -> Any:
data = await self.receive_str(timeout=timeout)
return loads(data) # type: ignore[arg-type]
def __aiter__(self) -> Self:
return self
@overload
async def __anext__(
self: "ClientWebSocketResponse[Literal[True]]",
) -> WSMessageDecodeText: ...
@overload
async def __anext__(
self: "ClientWebSocketResponse[Literal[False]]",
) -> WSMessageNoDecodeText: ...
@overload
async def __anext__(
self: "ClientWebSocketResponse[_DecodeText]",
) -> WSMessageDecodeText | WSMessageNoDecodeText: ...
async def __anext__(self) -> WSMessageDecodeText | WSMessageNoDecodeText:
msg = await self.receive()
if msg.type in (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.CLOSED):
raise StopAsyncIteration
return msg
async def __aenter__(self) -> Self:
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_val: BaseException | None,
exc_tb: TracebackType | None,
) -> None:
await self.close()
+447
View File
@@ -0,0 +1,447 @@
import asyncio
import sys
import zlib
from abc import ABC, abstractmethod
from concurrent.futures import Executor
from typing import Any, Final, Protocol, TypedDict, cast
if sys.version_info >= (3, 12):
from collections.abc import Buffer
else:
from typing import Union
Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"]
try:
try:
import brotlicffi as brotli
except ImportError:
import brotli
HAS_BROTLI = True
except ImportError: # pragma: no cover
HAS_BROTLI = False
try:
if sys.version_info >= (3, 14):
from compression.zstd import ZstdDecompressor # noqa: I900
else: # TODO(PY314): Remove mentions of backports.zstd across codebase
from backports.zstd import ZstdDecompressor
HAS_ZSTD = True
except ImportError:
HAS_ZSTD = False
MAX_SYNC_CHUNK_SIZE = 4096
# Unlimited decompression constants - different libraries use different conventions
ZLIB_MAX_LENGTH_UNLIMITED = 0 # zlib uses 0 to mean unlimited
ZSTD_MAX_LENGTH_UNLIMITED = -1 # zstd uses -1 to mean unlimited
class ZLibCompressObjProtocol(Protocol):
def compress(self, data: Buffer) -> bytes: ...
def flush(self, mode: int = ..., /) -> bytes: ...
class ZLibDecompressObjProtocol(Protocol):
def decompress(self, data: Buffer, max_length: int = ...) -> bytes: ...
def flush(self, length: int = ..., /) -> bytes: ...
@property
def eof(self) -> bool: ...
@property
def unconsumed_tail(self) -> bytes: ...
@property
def unused_data(self) -> bytes: ...
class ZLibBackendProtocol(Protocol):
MAX_WBITS: int
Z_FULL_FLUSH: int
Z_SYNC_FLUSH: int
Z_BEST_SPEED: int
Z_FINISH: int
def compressobj(
self,
level: int = ...,
method: int = ...,
wbits: int = ...,
memLevel: int = ...,
strategy: int = ...,
zdict: Buffer | None = ...,
) -> ZLibCompressObjProtocol: ...
def decompressobj(
self, wbits: int = ..., zdict: Buffer = ...
) -> ZLibDecompressObjProtocol: ...
def compress(
self, data: Buffer, /, level: int = ..., wbits: int = ...
) -> bytes: ...
def decompress(
self, data: Buffer, /, wbits: int = ..., bufsize: int = ...
) -> bytes: ...
class CompressObjArgs(TypedDict, total=False):
wbits: int
strategy: int
level: int
class ZLibBackendWrapper:
def __init__(self, _zlib_backend: ZLibBackendProtocol):
self._zlib_backend: ZLibBackendProtocol = _zlib_backend
@property
def name(self) -> str:
return getattr(self._zlib_backend, "__name__", "undefined")
@property
def MAX_WBITS(self) -> int:
return self._zlib_backend.MAX_WBITS
@property
def Z_FULL_FLUSH(self) -> int:
return self._zlib_backend.Z_FULL_FLUSH
@property
def Z_SYNC_FLUSH(self) -> int:
return self._zlib_backend.Z_SYNC_FLUSH
@property
def Z_BEST_SPEED(self) -> int:
return self._zlib_backend.Z_BEST_SPEED
@property
def Z_FINISH(self) -> int:
return self._zlib_backend.Z_FINISH
def compressobj(self, *args: Any, **kwargs: Any) -> ZLibCompressObjProtocol:
return self._zlib_backend.compressobj(*args, **kwargs)
def decompressobj(self, *args: Any, **kwargs: Any) -> ZLibDecompressObjProtocol:
return self._zlib_backend.decompressobj(*args, **kwargs)
def compress(self, data: Buffer, *args: Any, **kwargs: Any) -> bytes:
return self._zlib_backend.compress(data, *args, **kwargs)
def decompress(self, data: Buffer, *args: Any, **kwargs: Any) -> bytes:
return self._zlib_backend.decompress(data, *args, **kwargs)
# Everything not explicitly listed in the Protocol we just pass through
def __getattr__(self, attrname: str) -> Any:
return getattr(self._zlib_backend, attrname)
ZLibBackend: ZLibBackendWrapper = ZLibBackendWrapper(zlib)
def set_zlib_backend(new_zlib_backend: ZLibBackendProtocol) -> None:
ZLibBackend._zlib_backend = new_zlib_backend
def encoding_to_mode(
encoding: str | None = None,
suppress_deflate_header: bool = False,
) -> int:
if encoding == "gzip":
return 16 + ZLibBackend.MAX_WBITS
return -ZLibBackend.MAX_WBITS if suppress_deflate_header else ZLibBackend.MAX_WBITS
class DecompressionBaseHandler(ABC):
def __init__(
self,
executor: Executor | None = None,
max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE,
):
"""Base class for decompression handlers."""
self._executor = executor
self._max_sync_chunk_size = max_sync_chunk_size
@abstractmethod
def decompress_sync(
self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED
) -> bytes:
"""Decompress the given data."""
async def decompress(
self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED
) -> bytes:
"""Decompress the given data."""
if (
self._max_sync_chunk_size is not None
and len(data) > self._max_sync_chunk_size
):
return await asyncio.get_event_loop().run_in_executor(
self._executor, self.decompress_sync, data, max_length
)
return self.decompress_sync(data, max_length)
@property
@abstractmethod
def data_available(self) -> bool:
"""Return True if more output is available by passing b""."""
class ZLibCompressor:
def __init__(
self,
encoding: str | None = None,
suppress_deflate_header: bool = False,
level: int | None = None,
wbits: int | None = None,
strategy: int | None = None,
executor: Executor | None = None,
max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE,
):
self._executor = executor
self._max_sync_chunk_size = max_sync_chunk_size
self._mode = (
encoding_to_mode(encoding, suppress_deflate_header)
if wbits is None
else wbits
)
self._zlib_backend: Final = ZLibBackendWrapper(ZLibBackend._zlib_backend)
kwargs: CompressObjArgs = {}
kwargs["wbits"] = self._mode
if strategy is not None:
kwargs["strategy"] = strategy
if level is not None:
kwargs["level"] = level
self._compressor = self._zlib_backend.compressobj(**kwargs)
def compress_sync(self, data: Buffer) -> bytes:
return self._compressor.compress(data)
async def compress(self, data: Buffer) -> bytes:
"""Compress the data and returned the compressed bytes.
Note that flush() must be called after the last call to compress()
If the data size is large than the max_sync_chunk_size, the compression
will be done in the executor. Otherwise, the compression will be done
in the event loop.
**WARNING: This method is NOT cancellation-safe when used with flush().**
If this operation is cancelled, the compressor state may be corrupted.
The connection MUST be closed after cancellation to avoid data corruption
in subsequent compress operations.
For cancellation-safe compression (e.g., WebSocket), the caller MUST wrap
compress() + flush() + send operations in a shield and lock to ensure atomicity.
"""
# For large payloads, offload compression to executor to avoid blocking event loop
should_use_executor = (
self._max_sync_chunk_size is not None
and len(data) > self._max_sync_chunk_size
)
if should_use_executor:
return await asyncio.get_running_loop().run_in_executor(
self._executor, self._compressor.compress, data
)
return self.compress_sync(data)
def flush(self, mode: int | None = None) -> bytes:
"""Flush the compressor synchronously.
**WARNING: This method is NOT cancellation-safe when called after compress().**
The flush() operation accesses shared compressor state. If compress() was
cancelled, calling flush() may result in corrupted data. The connection MUST
be closed after compress() cancellation.
For cancellation-safe compression (e.g., WebSocket), the caller MUST wrap
compress() + flush() + send operations in a shield and lock to ensure atomicity.
"""
return self._compressor.flush(
mode if mode is not None else self._zlib_backend.Z_FINISH
)
class ZLibDecompressor(DecompressionBaseHandler):
def __init__(
self,
encoding: str | None = None,
suppress_deflate_header: bool = False,
executor: Executor | None = None,
max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE,
):
super().__init__(executor=executor, max_sync_chunk_size=max_sync_chunk_size)
self._mode = encoding_to_mode(encoding, suppress_deflate_header)
self._zlib_backend: Final = ZLibBackendWrapper(ZLibBackend._zlib_backend)
self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode)
self._last_empty = False
self._pending_unused_data: bytes | None = None
def decompress_sync(
self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED
) -> bytes:
if self._pending_unused_data is not None:
data = self._pending_unused_data + bytes(data)
self._pending_unused_data = None
result = self._decompressor.decompress(
self._decompressor.unconsumed_tail + data, max_length
)
# Only way to know that isal has no further data is checking we get no output
self._last_empty = result == b""
# Handle concatenated gzip/deflate streams (multi-member).
# After a member ends, unused_data holds the start of the next member.
# Create a fresh decompressor for each subsequent member.
while self._decompressor.eof and self._decompressor.unused_data:
unused = self._decompressor.unused_data
self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode)
if max_length != ZLIB_MAX_LENGTH_UNLIMITED:
max_length -= len(result)
if max_length <= 0:
self._pending_unused_data = unused
break
chunk = self._decompressor.decompress(unused, max_length)
self._last_empty = chunk == b""
result += chunk
# Member ended exactly at chunk boundary — no unused_data, but the
# next feed_data() call would fail on the spent decompressor.
# Only reset for gzip; deflate's feed_eof() relies on eof=True to
# confirm the stream is complete.
if self._decompressor.eof and self._mode > self._zlib_backend.MAX_WBITS:
self._decompressor = self._zlib_backend.decompressobj(wbits=self._mode)
return result
def flush(self, length: int = 0) -> bytes:
return (
self._decompressor.flush(length)
if length > 0
else self._decompressor.flush()
)
@property
def data_available(self) -> bool:
return (
bool(self._decompressor.unconsumed_tail)
or not self._last_empty
or self._pending_unused_data is not None
)
@property
def eof(self) -> bool:
return self._decompressor.eof
class BrotliDecompressor(DecompressionBaseHandler):
# Supports both 'brotlipy' and 'Brotli' packages
# since they share an import name. The top branches
# are for 'brotlipy' and bottom branches for 'Brotli'
def __init__(
self,
executor: Executor | None = None,
max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE,
) -> None:
"""Decompress data using the Brotli library."""
if not HAS_BROTLI:
raise RuntimeError(
"The brotli decompression is not available. "
"Please install `Brotli` module"
)
self._obj = brotli.Decompressor()
self._last_empty = False
super().__init__(executor=executor, max_sync_chunk_size=max_sync_chunk_size)
def decompress_sync(
self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED
) -> bytes:
"""Decompress the given data."""
if hasattr(self._obj, "decompress"):
if max_length == ZLIB_MAX_LENGTH_UNLIMITED:
result = cast(bytes, self._obj.decompress(data))
else:
result = cast(bytes, self._obj.decompress(data, max_length))
else:
if max_length == ZLIB_MAX_LENGTH_UNLIMITED:
result = cast(bytes, self._obj.process(data))
else:
result = cast(bytes, self._obj.process(data, max_length))
# Only way to know that brotli has no further data is checking we get no output
self._last_empty = result == b""
return result
def flush(self) -> bytes:
"""Flush the decompressor."""
if hasattr(self._obj, "flush"):
return cast(bytes, self._obj.flush())
return b""
@property
def data_available(self) -> bool:
return not self._obj.is_finished() and not self._last_empty
class ZSTDDecompressor(DecompressionBaseHandler):
def __init__(
self,
executor: Executor | None = None,
max_sync_chunk_size: int | None = MAX_SYNC_CHUNK_SIZE,
) -> None:
if not HAS_ZSTD:
raise RuntimeError(
"The zstd decompression is not available. "
"Please install `backports.zstd` module"
)
self._obj = ZstdDecompressor()
self._pending_unused_data: bytes | None = None
super().__init__(executor=executor, max_sync_chunk_size=max_sync_chunk_size)
def decompress_sync(
self, data: Buffer, max_length: int = ZLIB_MAX_LENGTH_UNLIMITED
) -> bytes:
# zstd uses -1 for unlimited, while zlib uses 0 for unlimited
# Convert the zlib convention (0=unlimited) to zstd convention (-1=unlimited)
zstd_max_length = (
ZSTD_MAX_LENGTH_UNLIMITED
if max_length == ZLIB_MAX_LENGTH_UNLIMITED
else max_length
)
if self._pending_unused_data is not None:
data = self._pending_unused_data + data
self._pending_unused_data = None
result = self._obj.decompress(data, zstd_max_length)
# Handle multi-frame zstd streams.
# https://datatracker.ietf.org/doc/html/rfc8878#section-3.1.1
# ZstdDecompressor handles one frame only. When a frame ends,
# eof becomes True and any trailing data goes to unused_data.
# We create a fresh decompressor to continue with the next frame.
while self._obj.eof and self._obj.unused_data:
unused_data = self._obj.unused_data
self._obj = ZstdDecompressor()
if zstd_max_length != ZSTD_MAX_LENGTH_UNLIMITED:
zstd_max_length -= len(result)
if zstd_max_length <= 0:
self._pending_unused_data = unused_data
break
result += self._obj.decompress(unused_data, zstd_max_length)
# Frame ended exactly at chunk boundary — no unused_data, but the
# next feed_data() call would fail on the spent decompressor.
# Prepare a fresh one for the next chunk.
if self._obj.eof:
self._obj = ZstdDecompressor()
return result
def flush(self) -> bytes:
return b""
@property
def data_available(self) -> bool:
return (
not self._obj.needs_input and not self._obj.eof
) or self._pending_unused_data is not None
File diff suppressed because it is too large Load Diff
+678
View File
@@ -0,0 +1,678 @@
import asyncio
import calendar
import contextlib
import datetime
import heapq
import itertools
import json
import os
import pathlib
import pickle
import re
import time
import warnings
from collections import defaultdict
from collections.abc import Iterable, Iterator, Mapping
from http.cookies import BaseCookie, Morsel, SimpleCookie
from types import MappingProxyType
from typing import Union
from yarl import URL
from ._cookie_helpers import preserve_morsel_with_coded_value
from .abc import AbstractCookieJar, ClearCookiePredicate
from .helpers import is_ip_address
from .typedefs import LooseCookies, PathLike, StrOrURL
__all__ = ("CookieJar", "DummyCookieJar")
CookieItem = Union[str, "Morsel[str]"]
# We cache these string methods here as their use is in performance critical code.
_FORMAT_PATH = "{}/{}".format
_FORMAT_DOMAIN_REVERSED = "{1}.{0}".format
# The minimum number of scheduled cookie expirations before we start cleaning up
# the expiration heap. This is a performance optimization to avoid cleaning up the
# heap too often when there are only a few scheduled expirations.
_MIN_SCHEDULED_COOKIE_EXPIRATION = 100
_SIMPLE_COOKIE = SimpleCookie()
# Not persisted; the absolute deadline is saved instead.
_RELATIVE_EXPIRY_ATTRS = frozenset(("max-age", "expires"))
class _RestrictedCookieUnpickler(pickle._Unpickler):
"""A restricted unpickler that only allows cookie-related types.
This prevents arbitrary code execution when loading pickled cookie data
from untrusted sources. Only types that are expected in a serialized
CookieJar are permitted.
Subclasses :class:`pickle._Unpickler` (the pure-Python implementation)
rather than :class:`pickle.Unpickler` because the accelerated unpickler
on some implementations (notably PyPy) does not dispatch through
:meth:`find_class` overrides.
See: https://docs.python.org/3/library/pickle.html#restricting-globals
"""
_ALLOWED_CLASSES: frozenset[tuple[str, str]] = frozenset(
{
# Core cookie types
("http.cookies", "SimpleCookie"),
("http.cookies", "Morsel"),
# Container types used by CookieJar._cookies
("collections", "defaultdict"),
# builtins that pickle uses for reconstruction
("builtins", "tuple"),
("builtins", "set"),
("builtins", "frozenset"),
("builtins", "dict"),
}
)
def find_class(self, module: str, name: str) -> type:
if (module, name) not in self._ALLOWED_CLASSES:
raise pickle.UnpicklingError(
f"Forbidden class: {module}.{name}. "
"CookieJar.load() only allows cookie-related types for security. "
"See https://docs.python.org/3/library/pickle.html#restricting-globals"
)
return super().find_class(module, name) # type: ignore[no-any-return]
class CookieJar(AbstractCookieJar):
"""Implements cookie storage adhering to RFC 6265."""
DATE_TOKENS_RE = re.compile(
r"[\x09\x20-\x2F\x3B-\x40\x5B-\x60\x7B-\x7E]*"
r"(?P<token>[\x00-\x08\x0A-\x1F\d:a-zA-Z\x7F-\xFF]+)"
)
DATE_HMS_TIME_RE = re.compile(r"(\d{1,2}):(\d{1,2}):(\d{1,2})")
DATE_DAY_OF_MONTH_RE = re.compile(r"(\d{1,2})")
DATE_MONTH_RE = re.compile(
"(jan)|(feb)|(mar)|(apr)|(may)|(jun)|(jul)|(aug)|(sep)|(oct)|(nov)|(dec)",
re.I,
)
DATE_YEAR_RE = re.compile(r"(\d{2,4})")
# calendar.timegm() fails for timestamps after datetime.datetime.max
# Minus one as a loss of precision occurs when timestamp() is called.
MAX_TIME = (
int(datetime.datetime.max.replace(tzinfo=datetime.timezone.utc).timestamp()) - 1
)
try:
calendar.timegm(time.gmtime(MAX_TIME))
except OSError:
# Hit the maximum representable time on Windows
# https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/localtime-localtime32-localtime64
MAX_TIME = calendar.timegm((3000, 12, 31, 23, 59, 59, -1, -1, -1))
except OverflowError:
# #4515: datetime.max may not be representable on 32-bit platforms
MAX_TIME = 2**31 - 1
# Avoid minuses in the future, 3x faster
SUB_MAX_TIME = MAX_TIME - 1
def __init__(
self,
*,
unsafe: bool = False,
quote_cookie: bool = True,
treat_as_secure_origin: StrOrURL | list[StrOrURL] | None = None,
loop: asyncio.AbstractEventLoop | None = None,
) -> None:
super().__init__(loop=loop)
self._cookies: defaultdict[tuple[str, str], SimpleCookie] = defaultdict(
SimpleCookie
)
self._morsel_cache: defaultdict[tuple[str, str], dict[str, Morsel[str]]] = (
defaultdict(dict)
)
self._host_only_cookies: set[tuple[str, str]] = set()
self._unsafe = unsafe
self._quote_cookie = quote_cookie
if treat_as_secure_origin is None:
treat_as_secure_origin = []
elif isinstance(treat_as_secure_origin, URL):
treat_as_secure_origin = [treat_as_secure_origin.origin()]
elif isinstance(treat_as_secure_origin, str):
treat_as_secure_origin = [URL(treat_as_secure_origin).origin()]
else:
treat_as_secure_origin = [
URL(url).origin() if isinstance(url, str) else url.origin()
for url in treat_as_secure_origin
]
self._treat_as_secure_origin = treat_as_secure_origin
self._expire_heap: list[tuple[float, tuple[str, str, str]]] = []
self._expirations: dict[tuple[str, str, str], float] = {}
@property
def unsafe(self) -> bool:
return self._unsafe
@property
def quote_cookie(self) -> bool:
return self._quote_cookie
@property
def cookies(self) -> MappingProxyType[tuple[str, str], SimpleCookie]:
"""Return the cookies stored in this jar."""
return MappingProxyType(self._cookies)
@property
def host_only_cookies(self) -> frozenset[tuple[str, str]]:
"""Return the host-only cookies stored in this jar."""
return frozenset(self._host_only_cookies)
def save(self, file_path: PathLike) -> None:
"""Save cookies to a file using JSON format.
:param file_path: Path to file where cookies will be serialized,
:class:`str` or :class:`pathlib.Path` instance.
"""
file_path = pathlib.Path(file_path)
data: dict[str, dict[str, dict[str, str | bool | float]]] = {}
for (domain, path), cookie in self._cookies.items():
key = f"{domain}|{path}"
data[key] = {}
for name, morsel in cookie.items():
morsel_data: dict[str, str | bool | float] = {
"key": morsel.key,
"value": morsel.value,
"coded_value": morsel.coded_value,
}
# Skip relative expiry; the absolute deadline is saved below.
for attr in morsel._reserved: # type: ignore[attr-defined]
if attr in _RELATIVE_EXPIRY_ATTRS:
continue
attr_val = morsel[attr]
if attr_val:
morsel_data[attr] = attr_val
# Persist or it reloads as a domain cookie and leaks to subdomains.
if (domain, name) in self._host_only_cookies:
morsel_data["host_only"] = True
if (exp := self._expirations.get((domain, path, name))) is not None:
morsel_data["expires_timestamp"] = exp
data[key][name] = morsel_data
# Cookie persistence may include authentication/session tokens.
# Use 0o600 at creation time to avoid umask-dependent overexposure
# and enforce least-privilege access to sensitive credential data.
with open(
file_path,
mode="w",
encoding="utf-8",
opener=lambda path, flags: os.open(path, flags, 0o600),
) as f:
json.dump(data, f, indent=2)
def load(self, file_path: PathLike) -> None:
"""Load cookies from a file.
Tries to load JSON format first. Falls back to loading legacy
pickle format (using a restricted unpickler) for backward
compatibility with existing cookie files.
Replaces the current jar contents; loaded cookies pass through the
same acceptance rules as :meth:`update_cookies`.
:param file_path: Path to file from where cookies will be
imported, :class:`str` or :class:`pathlib.Path` instance.
"""
file_path = pathlib.Path(file_path)
# Try JSON format first
try:
with file_path.open(mode="r", encoding="utf-8") as f:
data = json.load(f)
self._load_json_data(data)
except (json.JSONDecodeError, UnicodeDecodeError, ValueError):
# Fall back to legacy pickle format with restricted unpickler
with file_path.open(mode="rb") as f:
self._cookies = _RestrictedCookieUnpickler(f).load()
def _load_json_data(
self, data: dict[str, dict[str, dict[str, str | bool | float]]]
) -> None:
"""Replace contents, routing cookies through update_cookies()."""
self.clear()
for compound_key, cookie_data in data.items():
domain, path = compound_key.split("|", 1)
for name, morsel_data in cookie_data.items():
morsel: Morsel[str] = Morsel()
# Use __setstate__ to bypass validation, same pattern
# used in _build_morsel and _cookie_helpers.
morsel.__setstate__( # type: ignore[attr-defined]
{
"key": morsel_data["key"],
"value": morsel_data["value"],
"coded_value": morsel_data["coded_value"],
}
)
# Restore morsel attributes
for attr in morsel._reserved: # type: ignore[attr-defined]
if attr in morsel_data and attr not in (
"key",
"value",
"coded_value",
):
morsel[attr] = morsel_data[attr]
# Drop the domain so update_cookies() re-marks it host-only.
if morsel_data.get("host_only"):
morsel["domain"] = ""
response_url = (
URL.build(scheme="https", host=domain) if domain else URL()
)
self.update_cookies({name: morsel}, response_url)
# Restore the absolute deadline; update_cookies() schedules none.
if (exp := morsel_data.get("expires_timestamp")) is not None:
self._expire_cookie(float(exp), domain, path, name)
self._do_expiration()
def clear(self, predicate: ClearCookiePredicate | None = None) -> None:
if predicate is None:
self._expire_heap.clear()
self._cookies.clear()
self._morsel_cache.clear()
self._host_only_cookies.clear()
self._expirations.clear()
return
now = time.time()
to_del = [
key
for (domain, path), cookie in self._cookies.items()
for name, morsel in cookie.items()
if (
(key := (domain, path, name)) in self._expirations
and self._expirations[key] <= now
)
or predicate(morsel)
]
if to_del:
self._delete_cookies(to_del)
def clear_domain(self, domain: str) -> None:
self.clear(lambda x: self._is_domain_match(domain, x["domain"]))
def __iter__(self) -> "Iterator[Morsel[str]]":
self._do_expiration()
for val in self._cookies.values():
yield from val.values()
def __len__(self) -> int:
"""Return number of cookies.
This function does not iterate self to avoid unnecessary expiration
checks.
"""
return sum(len(cookie.values()) for cookie in self._cookies.values())
def _do_expiration(self) -> None:
"""Remove expired cookies."""
if not (expire_heap_len := len(self._expire_heap)):
return
# If the expiration heap grows larger than the number expirations
# times two, we clean it up to avoid keeping expired entries in
# the heap and consuming memory. We guard this with a minimum
# threshold to avoid cleaning up the heap too often when there are
# only a few scheduled expirations.
if (
expire_heap_len > _MIN_SCHEDULED_COOKIE_EXPIRATION
and expire_heap_len > len(self._expirations) * 2
):
# Remove any expired entries from the expiration heap
# that do not match the expiration time in the expirations
# as it means the cookie has been re-added to the heap
# with a different expiration time.
self._expire_heap = [
entry
for entry in self._expire_heap
if self._expirations.get(entry[1]) == entry[0]
]
heapq.heapify(self._expire_heap)
now = time.time()
to_del: list[tuple[str, str, str]] = []
# Find any expired cookies and add them to the to-delete list
while self._expire_heap:
when, cookie_key = self._expire_heap[0]
if when > now:
break
heapq.heappop(self._expire_heap)
# Check if the cookie hasn't been re-added to the heap
# with a different expiration time as it will be removed
# later when it reaches the top of the heap and its
# expiration time is met.
if self._expirations.get(cookie_key) == when:
to_del.append(cookie_key)
if to_del:
self._delete_cookies(to_del)
def _delete_cookies(self, to_del: list[tuple[str, str, str]]) -> None:
for domain, path, name in to_del:
self._host_only_cookies.discard((domain, name))
self._cookies[(domain, path)].pop(name, None)
self._morsel_cache[(domain, path)].pop(name, None)
self._expirations.pop((domain, path, name), None)
def _expire_cookie(self, when: float, domain: str, path: str, name: str) -> None:
cookie_key = (domain, path, name)
if self._expirations.get(cookie_key) == when:
# Avoid adding duplicates to the heap
return
heapq.heappush(self._expire_heap, (when, cookie_key))
self._expirations[cookie_key] = when
def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None:
"""Update cookies."""
hostname = response_url.raw_host
if not self._unsafe and is_ip_address(hostname):
# Don't accept cookies from IPs
return
if isinstance(cookies, Mapping):
cookies = cookies.items()
for name, cookie in cookies:
if not isinstance(cookie, Morsel):
tmp = SimpleCookie()
tmp[name] = cookie # type: ignore[assignment]
cookie = tmp[name]
domain = cookie["domain"]
# ignore domains with trailing dots
if domain and domain[-1] == ".":
domain = ""
del cookie["domain"]
if not domain and hostname is not None:
# Set the cookie's domain to the response hostname
# and set its host-only-flag
self._host_only_cookies.add((hostname, name))
domain = cookie["domain"] = hostname
if domain and domain[0] == ".":
# Remove leading dot
domain = domain[1:]
cookie["domain"] = domain
if hostname and not self._is_domain_match(domain, hostname):
# Setting cookies for different domains is not allowed
continue
path = cookie["path"]
if not path or path[0] != "/":
# Set the cookie's path to the response path
path = response_url.path
if not path.startswith("/"):
path = "/"
else:
# Cut everything from the last slash to the end
path = "/" + path[1 : path.rfind("/")]
cookie["path"] = path
path = path.rstrip("/")
if max_age := cookie["max-age"]:
try:
delta_seconds = int(max_age)
max_age_expiration = min(time.time() + delta_seconds, self.MAX_TIME)
self._expire_cookie(max_age_expiration, domain, path, name)
except ValueError:
cookie["max-age"] = ""
elif expires := cookie["expires"]:
if expire_time := self._parse_date(expires):
self._expire_cookie(expire_time, domain, path, name)
else:
cookie["expires"] = ""
key = (domain, path)
if self._cookies[key].get(name) != cookie:
# Don't blow away the cache if the same
# cookie gets set again
self._cookies[key][name] = cookie
self._morsel_cache[key].pop(name, None)
self._do_expiration()
def filter_cookies(self, request_url: URL = URL()) -> "BaseCookie[str]":
"""Returns this jar's cookies filtered by their attributes."""
# We always use BaseCookie now since all
# cookies set on on filtered are fully constructed
# Morsels, not just names and values.
filtered: BaseCookie[str] = BaseCookie()
if not self._cookies:
# Skip do_expiration() if there are no cookies.
return filtered
self._do_expiration()
if not self._cookies:
# Skip rest of function if no non-expired cookies.
return filtered
if type(request_url) is not URL:
warnings.warn(
"filter_cookies expects yarl.URL instances only,"
f"and will stop working in 4.x, got {type(request_url)}",
DeprecationWarning,
stacklevel=2,
)
request_url = URL(request_url)
hostname = request_url.raw_host or ""
is_not_secure = request_url.scheme not in ("https", "wss")
if is_not_secure and self._treat_as_secure_origin:
request_origin = URL()
with contextlib.suppress(ValueError):
request_origin = request_url.origin()
is_not_secure = request_origin not in self._treat_as_secure_origin
# Send shared cookie
key = ("", "")
for c in self._cookies[key].values():
# Check cache first
if c.key in self._morsel_cache[key]:
filtered[c.key] = self._morsel_cache[key][c.key]
continue
# Build and cache the morsel
mrsl_val = self._build_morsel(c)
self._morsel_cache[key][c.key] = mrsl_val
filtered[c.key] = mrsl_val
if is_ip_address(hostname):
if not self._unsafe:
return filtered
domains: Iterable[str] = (hostname,)
else:
# Get all the subdomains that might match a cookie (e.g. "foo.bar.com", "bar.com", "com")
domains = itertools.accumulate(
reversed(hostname.split(".")), _FORMAT_DOMAIN_REVERSED
)
# Get all the path prefixes that might match a cookie (e.g. "", "/foo", "/foo/bar")
paths = itertools.accumulate(request_url.path.split("/"), _FORMAT_PATH)
# Create every combination of (domain, path) pairs.
pairs = itertools.product(domains, paths)
path_len = len(request_url.path)
# Point 2: https://www.rfc-editor.org/rfc/rfc6265.html#section-5.4
for p in pairs:
if p not in self._cookies:
continue
for name, cookie in self._cookies[p].items():
domain = cookie["domain"]
if (domain, name) in self._host_only_cookies and domain != hostname:
continue
# Skip edge case when the cookie has a trailing slash but request doesn't.
if len(cookie["path"]) > path_len:
continue
if is_not_secure and cookie["secure"]:
continue
# We already built the Morsel so reuse it here
if name in self._morsel_cache[p]:
filtered[name] = self._morsel_cache[p][name]
continue
# Build and cache the morsel
mrsl_val = self._build_morsel(cookie)
self._morsel_cache[p][name] = mrsl_val
filtered[name] = mrsl_val
return filtered
def _build_morsel(self, cookie: Morsel[str]) -> Morsel[str]:
"""Build a morsel for sending, respecting quote_cookie setting."""
if self._quote_cookie and cookie.coded_value and cookie.coded_value[0] == '"':
return preserve_morsel_with_coded_value(cookie)
morsel: Morsel[str] = Morsel()
if self._quote_cookie:
value, coded_value = _SIMPLE_COOKIE.value_encode(cookie.value)
else:
coded_value = value = cookie.value
# We use __setstate__ instead of the public set() API because it allows us to
# bypass validation and set already validated state. This is more stable than
# setting protected attributes directly and unlikely to change since it would
# break pickling.
morsel.__setstate__({"key": cookie.key, "value": value, "coded_value": coded_value}) # type: ignore[attr-defined]
return morsel
@staticmethod
def _is_domain_match(domain: str, hostname: str) -> bool:
"""Implements domain matching adhering to RFC 6265."""
if hostname == domain:
return True
if not hostname.endswith(domain):
return False
non_matching = hostname[: -len(domain)]
if not non_matching.endswith("."):
return False
return not is_ip_address(hostname)
@classmethod
def _parse_date(cls, date_str: str) -> int | None:
"""Implements date string parsing adhering to RFC 6265."""
if not date_str:
return None
found_time = False
found_day = False
found_month = False
found_year = False
hour = minute = second = 0
day = 0
month = 0
year = 0
for token_match in cls.DATE_TOKENS_RE.finditer(date_str):
token = token_match.group("token")
if not found_time:
time_match = cls.DATE_HMS_TIME_RE.match(token)
if time_match:
found_time = True
hour, minute, second = (int(s) for s in time_match.groups())
continue
if not found_day:
day_match = cls.DATE_DAY_OF_MONTH_RE.match(token)
if day_match:
found_day = True
day = int(day_match.group())
continue
if not found_month:
month_match = cls.DATE_MONTH_RE.match(token)
if month_match:
found_month = True
assert month_match.lastindex is not None
month = month_match.lastindex
continue
if not found_year:
year_match = cls.DATE_YEAR_RE.match(token)
if year_match:
found_year = True
year = int(year_match.group())
if 70 <= year <= 99:
year += 1900
elif 0 <= year <= 69:
year += 2000
if False in (found_day, found_month, found_year, found_time):
return None
if not 1 <= day <= 31:
return None
if year < 1601 or hour > 23 or minute > 59 or second > 59:
return None
return calendar.timegm((year, month, day, hour, minute, second, -1, -1, -1))
class DummyCookieJar(AbstractCookieJar):
"""Implements a dummy cookie storage.
It can be used with the ClientSession when no cookie processing is needed.
"""
def __init__(self, *, loop: asyncio.AbstractEventLoop | None = None) -> None:
super().__init__(loop=loop)
def __iter__(self) -> "Iterator[Morsel[str]]":
while False:
yield None
def __len__(self) -> int:
return 0
@property
def unsafe(self) -> bool:
return False
@property
def quote_cookie(self) -> bool:
return True
@property
def cookies(self) -> MappingProxyType[tuple[str, str], SimpleCookie]:
"""Return an empty mapping."""
return MappingProxyType({})
@property
def host_only_cookies(self) -> frozenset[tuple[str, str]]:
"""Return an empty frozenset."""
return frozenset()
def clear(self, predicate: ClearCookiePredicate | None = None) -> None:
pass
def clear_domain(self, domain: str) -> None:
pass
def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None:
pass
def filter_cookies(self, request_url: URL) -> "BaseCookie[str]":
return SimpleCookie()
+184
View File
@@ -0,0 +1,184 @@
import io
import warnings
from collections.abc import Iterable
from typing import Any
from urllib.parse import urlencode
from multidict import MultiDict, MultiDictProxy
from . import hdrs, multipart, payload
from .helpers import guess_filename
from .http_writer import _safe_header
from .payload import Payload
__all__ = ("FormData",)
class FormData:
"""Helper class for form body generation.
Supports multipart/form-data and application/x-www-form-urlencoded.
"""
def __init__(
self,
fields: Iterable[Any] = (),
quote_fields: bool = True,
charset: str | None = None,
*,
default_to_multipart: bool = False,
) -> None:
self._writer = multipart.MultipartWriter("form-data")
self._fields: list[Any] = []
self._is_multipart = default_to_multipart
self._quote_fields = quote_fields
self._charset = charset
if isinstance(fields, dict):
fields = list(fields.items())
elif not isinstance(fields, (list, tuple)):
fields = (fields,)
self.add_fields(*fields)
@property
def is_multipart(self) -> bool:
return self._is_multipart
def add_field(
self,
name: str,
value: Any,
*,
content_type: str | None = None,
filename: str | None = None,
content_transfer_encoding: str | None = None,
) -> None:
if isinstance(value, io.IOBase):
self._is_multipart = True
elif isinstance(value, (bytes, bytearray, memoryview)):
msg = (
"In v4, passing bytes will no longer create a file field. "
"Please explicitly use the filename parameter or pass a BytesIO object."
)
if filename is None and content_transfer_encoding is None:
warnings.warn(msg, DeprecationWarning)
filename = name
_safe_header(name)
type_options: MultiDict[str] = MultiDict({"name": name})
if filename is not None and not isinstance(filename, str):
raise TypeError("filename must be an instance of str. Got: %s" % filename)
if filename is None and isinstance(value, io.IOBase):
filename = guess_filename(value, name)
if filename is not None:
_safe_header(filename)
type_options["filename"] = filename
self._is_multipart = True
headers = {}
if content_type is not None:
if not isinstance(content_type, str):
raise TypeError(
"content_type must be an instance of str. Got: %s" % content_type
)
_safe_header(content_type)
headers[hdrs.CONTENT_TYPE] = content_type
self._is_multipart = True
if content_transfer_encoding is not None:
if not isinstance(content_transfer_encoding, str):
raise TypeError(
"content_transfer_encoding must be an instance"
" of str. Got: %s" % content_transfer_encoding
)
msg = (
"content_transfer_encoding is deprecated. "
"To maintain compatibility with v4 please pass a BytesPayload."
)
warnings.warn(msg, DeprecationWarning)
self._is_multipart = True
self._fields.append((type_options, headers, value))
def add_fields(self, *fields: Any) -> None:
to_add = list(fields)
while to_add:
rec = to_add.pop(0)
if isinstance(rec, io.IOBase):
k = guess_filename(rec, "unknown")
self.add_field(k, rec) # type: ignore[arg-type]
elif isinstance(rec, (MultiDictProxy, MultiDict)):
to_add.extend(rec.items())
elif isinstance(rec, (list, tuple)) and len(rec) == 2:
k, fp = rec
self.add_field(k, fp)
else:
raise TypeError(
"Only io.IOBase, multidict and (name, file) "
"pairs allowed, use .add_field() for passing "
f"more complex parameters, got {rec!r}"
)
def _gen_form_urlencoded(self) -> payload.BytesPayload:
# form data (x-www-form-urlencoded)
data = []
for type_options, _, value in self._fields:
data.append((type_options["name"], value))
charset = self._charset if self._charset is not None else "utf-8"
if charset == "utf-8":
content_type = "application/x-www-form-urlencoded"
else:
content_type = "application/x-www-form-urlencoded; charset=%s" % charset
return payload.BytesPayload(
urlencode(data, doseq=True, encoding=charset).encode(),
content_type=content_type,
)
def _gen_form_data(self) -> multipart.MultipartWriter:
"""Encode a list of fields using the multipart/form-data MIME format"""
for dispparams, headers, value in self._fields:
try:
if hdrs.CONTENT_TYPE in headers:
part = payload.get_payload(
value,
content_type=headers[hdrs.CONTENT_TYPE],
headers=headers,
encoding=self._charset,
)
else:
part = payload.get_payload(
value, headers=headers, encoding=self._charset
)
except Exception as exc:
raise TypeError(
"Can not serialize value type: %r\n "
"headers: %r\n value: %r" % (type(value), headers, value)
) from exc
if dispparams:
part.set_content_disposition(
"form-data", quote_fields=self._quote_fields, **dispparams
)
# FIXME cgi.FieldStorage doesn't likes body parts with
# Content-Length which were sent via chunked transfer encoding
assert part.headers is not None
part.headers.popall(hdrs.CONTENT_LENGTH, None)
self._writer.append_payload(part)
self._fields.clear()
return self._writer
def __call__(self) -> Payload:
if self._is_multipart:
return self._gen_form_data()
else:
return self._gen_form_urlencoded()
+115
View File
@@ -0,0 +1,115 @@
"""HTTP Headers constants."""
# After changing the file content call ./tools/gen.py
# to regenerate the headers parser
import itertools
from typing import Final
from multidict import istr
METH_ANY: Final[str] = "*"
METH_CONNECT: Final[str] = "CONNECT"
METH_HEAD: Final[str] = "HEAD"
METH_GET: Final[str] = "GET"
METH_DELETE: Final[str] = "DELETE"
METH_OPTIONS: Final[str] = "OPTIONS"
METH_PATCH: Final[str] = "PATCH"
METH_POST: Final[str] = "POST"
METH_PUT: Final[str] = "PUT"
METH_TRACE: Final[str] = "TRACE"
METH_ALL: Final[set[str]] = {
METH_CONNECT,
METH_HEAD,
METH_GET,
METH_DELETE,
METH_OPTIONS,
METH_PATCH,
METH_POST,
METH_PUT,
METH_TRACE,
}
ACCEPT: Final[istr] = istr("Accept")
ACCEPT_CHARSET: Final[istr] = istr("Accept-Charset")
ACCEPT_ENCODING: Final[istr] = istr("Accept-Encoding")
ACCEPT_LANGUAGE: Final[istr] = istr("Accept-Language")
ACCEPT_RANGES: Final[istr] = istr("Accept-Ranges")
ACCESS_CONTROL_MAX_AGE: Final[istr] = istr("Access-Control-Max-Age")
ACCESS_CONTROL_ALLOW_CREDENTIALS: Final[istr] = istr("Access-Control-Allow-Credentials")
ACCESS_CONTROL_ALLOW_HEADERS: Final[istr] = istr("Access-Control-Allow-Headers")
ACCESS_CONTROL_ALLOW_METHODS: Final[istr] = istr("Access-Control-Allow-Methods")
ACCESS_CONTROL_ALLOW_ORIGIN: Final[istr] = istr("Access-Control-Allow-Origin")
ACCESS_CONTROL_EXPOSE_HEADERS: Final[istr] = istr("Access-Control-Expose-Headers")
ACCESS_CONTROL_REQUEST_HEADERS: Final[istr] = istr("Access-Control-Request-Headers")
ACCESS_CONTROL_REQUEST_METHOD: Final[istr] = istr("Access-Control-Request-Method")
AGE: Final[istr] = istr("Age")
ALLOW: Final[istr] = istr("Allow")
AUTHORIZATION: Final[istr] = istr("Authorization")
CACHE_CONTROL: Final[istr] = istr("Cache-Control")
CONNECTION: Final[istr] = istr("Connection")
CONTENT_DISPOSITION: Final[istr] = istr("Content-Disposition")
CONTENT_ENCODING: Final[istr] = istr("Content-Encoding")
CONTENT_LANGUAGE: Final[istr] = istr("Content-Language")
CONTENT_LENGTH: Final[istr] = istr("Content-Length")
CONTENT_LOCATION: Final[istr] = istr("Content-Location")
CONTENT_MD5: Final[istr] = istr("Content-MD5")
CONTENT_RANGE: Final[istr] = istr("Content-Range")
CONTENT_TRANSFER_ENCODING: Final[istr] = istr("Content-Transfer-Encoding")
CONTENT_TYPE: Final[istr] = istr("Content-Type")
COOKIE: Final[istr] = istr("Cookie")
DATE: Final[istr] = istr("Date")
DESTINATION: Final[istr] = istr("Destination")
DIGEST: Final[istr] = istr("Digest")
ETAG: Final[istr] = istr("Etag")
EXPECT: Final[istr] = istr("Expect")
EXPIRES: Final[istr] = istr("Expires")
FORWARDED: Final[istr] = istr("Forwarded")
FROM: Final[istr] = istr("From")
HOST: Final[istr] = istr("Host")
IF_MATCH: Final[istr] = istr("If-Match")
IF_MODIFIED_SINCE: Final[istr] = istr("If-Modified-Since")
IF_NONE_MATCH: Final[istr] = istr("If-None-Match")
IF_RANGE: Final[istr] = istr("If-Range")
IF_UNMODIFIED_SINCE: Final[istr] = istr("If-Unmodified-Since")
KEEP_ALIVE: Final[istr] = istr("Keep-Alive")
LAST_EVENT_ID: Final[istr] = istr("Last-Event-ID")
LAST_MODIFIED: Final[istr] = istr("Last-Modified")
LINK: Final[istr] = istr("Link")
LOCATION: Final[istr] = istr("Location")
MAX_FORWARDS: Final[istr] = istr("Max-Forwards")
ORIGIN: Final[istr] = istr("Origin")
PRAGMA: Final[istr] = istr("Pragma")
PROXY_AUTHENTICATE: Final[istr] = istr("Proxy-Authenticate")
PROXY_AUTHORIZATION: Final[istr] = istr("Proxy-Authorization")
RANGE: Final[istr] = istr("Range")
REFERER: Final[istr] = istr("Referer")
RETRY_AFTER: Final[istr] = istr("Retry-After")
SEC_WEBSOCKET_ACCEPT: Final[istr] = istr("Sec-WebSocket-Accept")
SEC_WEBSOCKET_VERSION: Final[istr] = istr("Sec-WebSocket-Version")
SEC_WEBSOCKET_PROTOCOL: Final[istr] = istr("Sec-WebSocket-Protocol")
SEC_WEBSOCKET_EXTENSIONS: Final[istr] = istr("Sec-WebSocket-Extensions")
SEC_WEBSOCKET_KEY: Final[istr] = istr("Sec-WebSocket-Key")
SEC_WEBSOCKET_KEY1: Final[istr] = istr("Sec-WebSocket-Key1")
SERVER: Final[istr] = istr("Server")
SET_COOKIE: Final[istr] = istr("Set-Cookie")
TE: Final[istr] = istr("TE")
TRAILER: Final[istr] = istr("Trailer")
TRANSFER_ENCODING: Final[istr] = istr("Transfer-Encoding")
UPGRADE: Final[istr] = istr("Upgrade")
URI: Final[istr] = istr("URI")
USER_AGENT: Final[istr] = istr("User-Agent")
VARY: Final[istr] = istr("Vary")
VIA: Final[istr] = istr("Via")
WANT_DIGEST: Final[istr] = istr("Want-Digest")
WARNING: Final[istr] = istr("Warning")
WWW_AUTHENTICATE: Final[istr] = istr("WWW-Authenticate")
X_FORWARDED_FOR: Final[istr] = istr("X-Forwarded-For")
X_FORWARDED_HOST: Final[istr] = istr("X-Forwarded-Host")
X_FORWARDED_PROTO: Final[istr] = istr("X-Forwarded-Proto")
# Case permutations of the Host header — for callers that match against
# raw header tokens before istr/CIMultiDict folding.
HOST_ALL: Final = frozenset(
map("".join, itertools.product(*zip(HOST.upper(), HOST.lower())))
)
File diff suppressed because it is too large Load Diff
+78
View File
@@ -0,0 +1,78 @@
import sys
from collections.abc import Mapping
from http import HTTPStatus
from . import __version__
from .http_exceptions import HttpProcessingError as HttpProcessingError
from .http_parser import (
HeadersParser as HeadersParser,
HttpParser as HttpParser,
HttpRequestParser as HttpRequestParser,
HttpResponseParser as HttpResponseParser,
RawRequestMessage as RawRequestMessage,
RawResponseMessage as RawResponseMessage,
)
from .http_websocket import (
WS_CLOSED_MESSAGE as WS_CLOSED_MESSAGE,
WS_CLOSING_MESSAGE as WS_CLOSING_MESSAGE,
WS_KEY as WS_KEY,
WebSocketError as WebSocketError,
WebSocketReader as WebSocketReader,
WebSocketWriter as WebSocketWriter,
WSCloseCode as WSCloseCode,
WSMessage as WSMessage,
WSMessageDecodeText as WSMessageDecodeText,
WSMessageNoDecodeText as WSMessageNoDecodeText,
WSMessageTextBytes as WSMessageTextBytes,
WSMsgType as WSMsgType,
ws_ext_gen as ws_ext_gen,
ws_ext_parse as ws_ext_parse,
)
from .http_writer import (
HttpVersion as HttpVersion,
HttpVersion10 as HttpVersion10,
HttpVersion11 as HttpVersion11,
StreamWriter as StreamWriter,
)
__all__ = (
"HttpProcessingError",
"RESPONSES",
"SERVER_SOFTWARE",
# .http_writer
"StreamWriter",
"HttpVersion",
"HttpVersion10",
"HttpVersion11",
# .http_parser
"HeadersParser",
"HttpParser",
"HttpRequestParser",
"HttpResponseParser",
"RawRequestMessage",
"RawResponseMessage",
# .http_websocket
"WS_CLOSED_MESSAGE",
"WS_CLOSING_MESSAGE",
"WS_KEY",
"WebSocketReader",
"WebSocketWriter",
"ws_ext_gen",
"ws_ext_parse",
"WSMessage",
"WSMessageDecodeText",
"WSMessageNoDecodeText",
"WSMessageTextBytes",
"WebSocketError",
"WSMsgType",
"WSCloseCode",
)
SERVER_SOFTWARE: str = (
f"Python/{sys.version_info[0]}.{sys.version_info[1]} aiohttp/{__version__}"
)
RESPONSES: Mapping[int, tuple[str, str]] = {
v: (v.phrase, v.description) for v in HTTPStatus.__members__.values()
}
+118
View File
@@ -0,0 +1,118 @@
"""Low-level http related exceptions."""
from textwrap import indent
from .typedefs import _CIMultiDict
__all__ = ("HttpProcessingError",)
class HttpProcessingError(Exception):
"""HTTP error.
Shortcut for raising HTTP errors with custom code, message and headers.
code: HTTP Error code.
message: (optional) Error message.
headers: (optional) Headers to be sent in response, a list of pairs
"""
code = 0
message = ""
headers = None
def __init__(
self,
*,
code: int | None = None,
message: str = "",
headers: _CIMultiDict | None = None,
) -> None:
if code is not None:
self.code = code
self.headers = headers
self.message = message
def __str__(self) -> str:
msg = indent(self.message, " ")
return f"{self.code}, message:\n{msg}"
def __repr__(self) -> str:
return f"<{self.__class__.__name__}: {self.code}, message={self.message!r}>"
class BadHttpMessage(HttpProcessingError):
code = 400
message = "Bad Request"
def __init__(self, message: str, *, headers: _CIMultiDict | None = None) -> None:
super().__init__(message=message, headers=headers)
self.args = (message,)
class HttpBadRequest(BadHttpMessage):
code = 400
message = "Bad Request"
class PayloadEncodingError(BadHttpMessage):
"""Base class for payload errors"""
class ContentEncodingError(PayloadEncodingError):
"""Content encoding error."""
class TransferEncodingError(PayloadEncodingError):
"""transfer encoding error."""
class ContentLengthError(PayloadEncodingError):
"""Not enough data to satisfy content length header."""
class DecompressSizeError(PayloadEncodingError):
"""Deprecated. Removed in v4."""
class LineTooLong(BadHttpMessage):
def __init__(
self,
line: str | bytes,
limit: str | int = "Unknown",
actual_size: str = "Unknown",
) -> None:
super().__init__(f"Got more than {limit} bytes when reading: {line!r}.")
self.args = (line, limit, actual_size)
class InvalidHeader(BadHttpMessage):
def __init__(self, hdr: bytes | str) -> None:
hdr_s = hdr.decode(errors="backslashreplace") if isinstance(hdr, bytes) else hdr
super().__init__(f"Invalid HTTP header: {hdr!r}")
self.hdr = hdr_s
self.args = (hdr,)
class BadStatusLine(BadHttpMessage):
def __init__(self, line: str = "", error: str | None = None) -> None:
if not isinstance(line, str):
line = repr(line)
super().__init__(error or f"Bad status line {line!r}")
self.args = (line,)
self.line = line
class BadHttpMethod(BadStatusLine):
"""Invalid HTTP method in status line."""
def __init__(self, line: str = "", error: str | None = None) -> None:
if error is None and line.startswith("\x16\x03"):
error = "Received HTTPS traffic on an HTTP port"
super().__init__(line, error or f"Bad HTTP method in status line {line!r}")
class InvalidURLError(BadHttpMessage):
pass
File diff suppressed because it is too large Load Diff
+41
View File
@@ -0,0 +1,41 @@
"""WebSocket protocol versions 13 and 8."""
from ._websocket.helpers import WS_KEY, ws_ext_gen, ws_ext_parse
from ._websocket.models import (
WS_CLOSED_MESSAGE,
WS_CLOSING_MESSAGE,
WebSocketError,
WSCloseCode,
WSHandshakeError,
WSMessage,
WSMessageDecodeText,
WSMessageNoDecodeText,
WSMessageTextBytes,
WSMsgType,
)
from ._websocket.reader import WebSocketReader
from ._websocket.writer import WebSocketWriter
# Messages that the WebSocketResponse.receive needs to handle internally
_INTERNAL_RECEIVE_TYPES = frozenset(
(WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.PING, WSMsgType.PONG)
)
__all__ = (
"WS_CLOSED_MESSAGE",
"WS_CLOSING_MESSAGE",
"WS_KEY",
"WebSocketReader",
"WebSocketWriter",
"WSMessage",
"WSMessageDecodeText",
"WSMessageNoDecodeText",
"WSMessageTextBytes",
"WebSocketError",
"WSMsgType",
"WSCloseCode",
"ws_ext_gen",
"ws_ext_parse",
"WSHandshakeError",
)
+381
View File
@@ -0,0 +1,381 @@
"""Http related parsers and protocol."""
import asyncio
import re
import sys
from typing import ( # noqa
TYPE_CHECKING,
Any,
Awaitable,
Callable,
Iterable,
List,
NamedTuple,
Optional,
)
from multidict import CIMultiDict
from .abc import AbstractStreamWriter
from .base_protocol import BaseProtocol
from .client_exceptions import ClientConnectionResetError
from .compression_utils import ZLibCompressor
from .helpers import NO_EXTENSIONS
__all__ = ("StreamWriter", "HttpVersion", "HttpVersion10", "HttpVersion11")
if sys.version_info >= (3, 12):
from collections.abc import Buffer
else:
from typing import Union
Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"]
MIN_PAYLOAD_FOR_WRITELINES = 2048
IS_PY313_BEFORE_313_2 = (3, 13, 0) <= sys.version_info < (3, 13, 2)
IS_PY_BEFORE_312_9 = sys.version_info < (3, 12, 9)
SKIP_WRITELINES = IS_PY313_BEFORE_313_2 or IS_PY_BEFORE_312_9
# writelines is not safe for use
# on Python 3.12+ until 3.12.9
# on Python 3.13+ until 3.13.2
# and on older versions it not any faster than write
# CVE-2024-12254: https://github.com/python/cpython/pull/127656
class HttpVersion(NamedTuple):
major: int
minor: int
HttpVersion10 = HttpVersion(1, 0)
HttpVersion11 = HttpVersion(1, 1)
_T_OnChunkSent = Optional[Callable[[Buffer], Awaitable[None]]]
_T_OnHeadersSent = Optional[Callable[["CIMultiDict[str]"], Awaitable[None]]]
class StreamWriter(AbstractStreamWriter):
length: int | None = None
chunked: bool = False
_eof: bool = False
_compress: ZLibCompressor | None = None
def __init__(
self,
protocol: BaseProtocol,
loop: asyncio.AbstractEventLoop,
on_chunk_sent: _T_OnChunkSent = None,
on_headers_sent: _T_OnHeadersSent = None,
) -> None:
self._protocol = protocol
self.loop = loop
self._on_chunk_sent: _T_OnChunkSent = on_chunk_sent
self._on_headers_sent: _T_OnHeadersSent = on_headers_sent
self._headers_buf: bytes | None = None
self._headers_written: bool = False
@property
def transport(self) -> asyncio.Transport | None:
return self._protocol.transport
@property
def protocol(self) -> BaseProtocol:
return self._protocol
def enable_chunking(self) -> None:
self.chunked = True
def enable_compression(
self, encoding: str = "deflate", strategy: int | None = None
) -> None:
self._compress = ZLibCompressor(encoding=encoding, strategy=strategy)
def _write(self, chunk: Buffer) -> None:
size = len(chunk)
self.buffer_size += size
self.output_size += size
transport = self._protocol.transport
if transport is None or transport.is_closing():
raise ClientConnectionResetError("Cannot write to closing transport")
transport.write(chunk)
def _writelines(self, chunks: Iterable[Buffer]) -> None:
size = 0
for chunk in chunks:
size += len(chunk)
self.buffer_size += size
self.output_size += size
transport = self._protocol.transport
if transport is None or transport.is_closing():
raise ClientConnectionResetError("Cannot write to closing transport")
if SKIP_WRITELINES or size < MIN_PAYLOAD_FOR_WRITELINES:
transport.write(b"".join(chunks))
else:
transport.writelines(chunks)
def _write_chunked_payload(self, chunk: Buffer) -> None:
"""Write a chunk with proper chunked encoding."""
chunk_len_pre = f"{len(chunk):x}\r\n".encode("ascii")
self._writelines((chunk_len_pre, chunk, b"\r\n"))
def _send_headers_with_payload(self, chunk: Buffer, is_eof: bool) -> None:
"""Send buffered headers with payload, coalescing into single write."""
# Mark headers as written
self._headers_written = True
headers_buf = self._headers_buf
self._headers_buf = None
if TYPE_CHECKING:
# Safe because callers (write() and write_eof()) only invoke this method
# after checking that self._headers_buf is truthy
assert headers_buf is not None
if not self.chunked:
# Non-chunked: coalesce headers with body
if chunk:
self._writelines((headers_buf, chunk))
else:
self._write(headers_buf)
return
# Coalesce headers with chunked data
if chunk:
chunk_len_pre = f"{len(chunk):x}\r\n".encode("ascii")
if is_eof:
self._writelines((headers_buf, chunk_len_pre, chunk, b"\r\n0\r\n\r\n"))
else:
self._writelines((headers_buf, chunk_len_pre, chunk, b"\r\n"))
elif is_eof:
self._writelines((headers_buf, b"0\r\n\r\n"))
else:
self._write(headers_buf)
async def write(
self, chunk: Buffer, *, drain: bool = True, LIMIT: int = 0x10000
) -> None:
"""
Writes chunk of data to a stream.
write_eof() indicates end of stream.
writer can't be used after write_eof() method being called.
write() return drain future.
"""
if self._on_chunk_sent is not None:
await self._on_chunk_sent(chunk)
if isinstance(chunk, memoryview):
if chunk.nbytes != len(chunk):
# just reshape it
chunk = chunk.cast("c")
if self._compress is not None:
chunk = await self._compress.compress(chunk)
if not chunk:
return
if self.length is not None:
chunk_len = len(chunk)
if self.length >= chunk_len:
self.length = self.length - chunk_len
else:
chunk = chunk[: self.length]
self.length = 0
if not chunk:
return
# Handle buffered headers for small payload optimization
if self._headers_buf and not self._headers_written:
self._send_headers_with_payload(chunk, False)
if drain and self.buffer_size > LIMIT:
self.buffer_size = 0
await self.drain()
return
if chunk:
if self.chunked:
self._write_chunked_payload(chunk)
else:
self._write(chunk)
if drain and self.buffer_size > LIMIT:
self.buffer_size = 0
await self.drain()
async def write_headers(
self, status_line: str, headers: "CIMultiDict[str]"
) -> None:
"""Write headers to the stream."""
if self._on_headers_sent is not None:
await self._on_headers_sent(headers)
# status + headers
buf = _serialize_headers(status_line, headers)
self._headers_written = False
self._headers_buf = buf
def send_headers(self) -> None:
"""Force sending buffered headers if not already sent."""
if not self._headers_buf or self._headers_written:
return
self._headers_written = True
headers_buf = self._headers_buf
self._headers_buf = None
if TYPE_CHECKING:
# Safe because we only enter this block when self._headers_buf is truthy
assert headers_buf is not None
self._write(headers_buf)
def set_eof(self) -> None:
"""Indicate that the message is complete."""
if self._eof:
return
# If headers haven't been sent yet, send them now
# This handles the case where there's no body at all
if self._headers_buf and not self._headers_written:
self._headers_written = True
headers_buf = self._headers_buf
self._headers_buf = None
if TYPE_CHECKING:
# Safe because we only enter this block when self._headers_buf is truthy
assert headers_buf is not None
# Combine headers and chunked EOF marker in a single write
if self.chunked:
self._writelines((headers_buf, b"0\r\n\r\n"))
else:
self._write(headers_buf)
elif self.chunked and self._headers_written:
# Headers already sent, just send the final chunk marker
self._write(b"0\r\n\r\n")
self._eof = True
async def write_eof(self, chunk: bytes = b"") -> None:
if self._eof:
return
if chunk and self._on_chunk_sent is not None:
await self._on_chunk_sent(chunk)
# Handle body/compression
if self._compress:
chunks: list[bytes] = []
chunks_len = 0
if chunk and (compressed_chunk := await self._compress.compress(chunk)):
chunks_len = len(compressed_chunk)
chunks.append(compressed_chunk)
flush_chunk = self._compress.flush()
chunks_len += len(flush_chunk)
chunks.append(flush_chunk)
assert chunks_len
# Send buffered headers with compressed data if not yet sent
if self._headers_buf and not self._headers_written:
self._headers_written = True
headers_buf = self._headers_buf
self._headers_buf = None
if self.chunked:
# Coalesce headers with compressed chunked data
chunk_len_pre = f"{chunks_len:x}\r\n".encode("ascii")
self._writelines(
(headers_buf, chunk_len_pre, *chunks, b"\r\n0\r\n\r\n")
)
else:
# Coalesce headers with compressed data
self._writelines((headers_buf, *chunks))
await self.drain()
self._eof = True
return
# Headers already sent, just write compressed data
if self.chunked:
chunk_len_pre = f"{chunks_len:x}\r\n".encode("ascii")
self._writelines((chunk_len_pre, *chunks, b"\r\n0\r\n\r\n"))
elif len(chunks) > 1:
self._writelines(chunks)
else:
self._write(chunks[0])
await self.drain()
self._eof = True
return
# No compression - send buffered headers if not yet sent
if self._headers_buf and not self._headers_written:
# Use helper to send headers with payload
self._send_headers_with_payload(chunk, True)
await self.drain()
self._eof = True
return
# Handle remaining body
if self.chunked:
if chunk:
# Write final chunk with EOF marker
self._writelines(
(f"{len(chunk):x}\r\n".encode("ascii"), chunk, b"\r\n0\r\n\r\n")
)
else:
self._write(b"0\r\n\r\n")
await self.drain()
self._eof = True
return
if chunk:
self._write(chunk)
await self.drain()
self._eof = True
async def drain(self) -> None:
"""Flush the write buffer.
The intended use is to write
await w.write(data)
await w.drain()
"""
protocol = self._protocol
if protocol.transport is not None and protocol._paused:
await protocol._drain_helper()
# https://www.rfc-editor.org/info/rfc9110/#section-5.5-5
# https://www.rfc-editor.org/info/rfc9112/#section-4-3
_FORBIDDEN_HEADER_CHARS_RE = re.compile(r"[\x00-\x08\x0a-\x1f\x7f]")
def _safe_header(string: str) -> str:
if _FORBIDDEN_HEADER_CHARS_RE.search(string) is not None:
raise ValueError(
"Forbidden control character detected in headers. "
"Potential header injection attack."
)
return string
def _py_serialize_headers(status_line: str, headers: "CIMultiDict[str]") -> bytes:
_safe_header(status_line)
headers_gen = (_safe_header(k) + ": " + _safe_header(v) for k, v in headers.items())
line = status_line + "\r\n" + "\r\n".join(headers_gen) + "\r\n\r\n"
return line.encode("utf-8")
_serialize_headers = _py_serialize_headers
try:
import aiohttp._http_writer as _http_writer # type: ignore[import-not-found]
_c_serialize_headers = _http_writer._serialize_headers
if not NO_EXTENSIONS:
_serialize_headers = _c_serialize_headers
except ImportError:
pass
+8
View File
@@ -0,0 +1,8 @@
import logging
access_logger = logging.getLogger("aiohttp.access")
client_logger = logging.getLogger("aiohttp.client")
internal_logger = logging.getLogger("aiohttp.internal")
server_logger = logging.getLogger("aiohttp.server")
web_logger = logging.getLogger("aiohttp.web")
ws_logger = logging.getLogger("aiohttp.websocket")
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+79
View File
@@ -0,0 +1,79 @@
"""
Payload implementation for coroutines as data provider.
As a simple case, you can upload data from file::
@aiohttp.streamer
async def file_sender(writer, file_name=None):
with open(file_name, 'rb') as f:
chunk = f.read(2**16)
while chunk:
await writer.write(chunk)
chunk = f.read(2**16)
Then you can use `file_sender` like this:
async with session.post('http://httpbin.org/post',
data=file_sender(file_name='huge_file')) as resp:
print(await resp.text())
..note:: Coroutine must accept `writer` as first argument
"""
import types
import warnings
from collections.abc import Awaitable, Callable
from typing import Any
from .abc import AbstractStreamWriter
from .payload import Payload, payload_type
__all__ = ("streamer",)
class _stream_wrapper:
def __init__(
self,
coro: Callable[..., Awaitable[None]],
args: tuple[Any, ...],
kwargs: dict[str, Any],
) -> None:
self.coro = types.coroutine(coro)
self.args = args
self.kwargs = kwargs
async def __call__(self, writer: AbstractStreamWriter) -> None:
await self.coro(writer, *self.args, **self.kwargs)
class streamer:
def __init__(self, coro: Callable[..., Awaitable[None]]) -> None:
warnings.warn(
"@streamer is deprecated, use async generators instead",
DeprecationWarning,
stacklevel=2,
)
self.coro = coro
def __call__(self, *args: Any, **kwargs: Any) -> _stream_wrapper:
return _stream_wrapper(self.coro, args, kwargs)
@payload_type(_stream_wrapper)
class StreamWrapperPayload(Payload):
async def write(self, writer: AbstractStreamWriter) -> None:
await self._value(writer)
def decode(self, encoding: str = "utf-8", errors: str = "strict") -> str:
raise TypeError("Unable to decode.")
@payload_type(streamer)
class StreamPayload(StreamWrapperPayload):
def __init__(self, value: Any, *args: Any, **kwargs: Any) -> None:
super().__init__(value(), *args, **kwargs)
async def write(self, writer: AbstractStreamWriter) -> None:
await self._value(writer)
+1
View File
@@ -0,0 +1 @@
Marker
+439
View File
@@ -0,0 +1,439 @@
import asyncio
import contextlib
import inspect
import warnings
from collections.abc import Awaitable, Callable, Iterator
from typing import Any, Protocol, overload
import pytest
from .test_utils import (
BaseTestServer,
RawTestServer,
TestClient,
TestServer,
loop_context,
setup_test_loop,
teardown_test_loop,
unused_port as _unused_port,
)
from .web import Application, BaseRequest, Request
from .web_protocol import _RequestHandler
try:
import uvloop
except ImportError: # pragma: no cover
uvloop = None # type: ignore[assignment]
class AiohttpClient(Protocol):
@overload
async def __call__(
self,
__param: Application,
*,
server_kwargs: dict[str, Any] | None = None,
**kwargs: Any,
) -> TestClient[Request, Application]: ...
@overload
async def __call__(
self,
__param: BaseTestServer,
*,
server_kwargs: dict[str, Any] | None = None,
**kwargs: Any,
) -> TestClient[BaseRequest, None]: ...
class AiohttpServer(Protocol):
def __call__(
self, app: Application, *, port: int | None = None, **kwargs: Any
) -> Awaitable[TestServer]: ...
class AiohttpRawServer(Protocol):
def __call__(
self, handler: _RequestHandler, *, port: int | None = None, **kwargs: Any
) -> Awaitable[RawTestServer]: ...
def pytest_addoption(parser): # type: ignore[no-untyped-def]
parser.addoption(
"--aiohttp-fast",
action="store_true",
default=False,
help="run tests faster by disabling extra checks",
)
parser.addoption(
"--aiohttp-loop",
action="store",
default="pyloop",
help="run tests with specific loop: pyloop, uvloop or all",
)
parser.addoption(
"--aiohttp-enable-loop-debug",
action="store_true",
default=False,
help="enable event loop debug mode",
)
def pytest_fixture_setup(fixturedef): # type: ignore[no-untyped-def]
"""Set up pytest fixture.
Allow fixtures to be coroutines. Run coroutine fixtures in an event loop.
"""
func = fixturedef.func
if inspect.isasyncgenfunction(func):
# async generator fixture
is_async_gen = True
elif inspect.iscoroutinefunction(func):
# regular async fixture
is_async_gen = False
else:
# not an async fixture, nothing to do
return
strip_request = False
if "request" not in fixturedef.argnames:
fixturedef.argnames += ("request",)
strip_request = True
def wrapper(*args, **kwargs): # type: ignore[no-untyped-def]
request = kwargs["request"]
if strip_request:
del kwargs["request"]
# if neither the fixture nor the test use the 'loop' fixture,
# 'getfixturevalue' will fail because the test is not parameterized
# (this can be removed someday if 'loop' is no longer parameterized)
if "loop" not in request.fixturenames:
raise Exception(
"Asynchronous fixtures must depend on the 'loop' fixture or "
"be used in tests depending from it."
)
_loop = request.getfixturevalue("loop")
if is_async_gen:
# for async generators, we need to advance the generator once,
# then advance it again in a finalizer
gen = func(*args, **kwargs)
def finalizer(): # type: ignore[no-untyped-def]
try:
return _loop.run_until_complete(gen.__anext__())
except StopAsyncIteration:
pass
request.addfinalizer(finalizer)
return _loop.run_until_complete(gen.__anext__())
else:
return _loop.run_until_complete(func(*args, **kwargs))
fixturedef.func = wrapper
@pytest.fixture
def fast(request): # type: ignore[no-untyped-def]
"""--fast config option"""
return request.config.getoption("--aiohttp-fast")
@pytest.fixture
def loop_debug(request): # type: ignore[no-untyped-def]
"""--enable-loop-debug config option"""
return request.config.getoption("--aiohttp-enable-loop-debug")
@contextlib.contextmanager
def _runtime_warning_context(): # type: ignore[no-untyped-def]
"""Context manager which checks for RuntimeWarnings.
This exists specifically to
avoid "coroutine 'X' was never awaited" warnings being missed.
If RuntimeWarnings occur in the context a RuntimeError is raised.
"""
with warnings.catch_warnings(record=True) as _warnings:
yield
rw = [
f"{w.filename}:{w.lineno}:{w.message}"
for w in _warnings
if w.category == RuntimeWarning
]
if rw:
raise RuntimeError(
"{} Runtime Warning{},\n{}".format(
len(rw), "" if len(rw) == 1 else "s", "\n".join(rw)
)
)
@contextlib.contextmanager
def _passthrough_loop_context(loop, fast=False): # type: ignore[no-untyped-def]
"""Passthrough loop context.
Sets up and tears down a loop unless one is passed in via the loop
argument when it's passed straight through.
"""
if loop:
# loop already exists, pass it straight through
yield loop
else:
# this shadows loop_context's standard behavior
loop = setup_test_loop()
yield loop
teardown_test_loop(loop, fast=fast)
def pytest_pycollect_makeitem(collector, name, obj): # type: ignore[no-untyped-def]
"""Fix pytest collecting for coroutines."""
if collector.funcnamefilter(name) and inspect.iscoroutinefunction(obj):
return list(collector._genfunctions(name, obj))
def pytest_pyfunc_call(pyfuncitem): # type: ignore[no-untyped-def]
"""Run coroutines in an event loop instead of a normal function call."""
fast = pyfuncitem.config.getoption("--aiohttp-fast")
if inspect.iscoroutinefunction(pyfuncitem.function):
warnings.warn(
"aiohttp.pytest_plugin will be removed in v4. Please install pytest-aiohttp.",
DeprecationWarning,
)
existing_loop = (
pyfuncitem.funcargs.get("proactor_loop")
or pyfuncitem.funcargs.get("selector_loop")
or pyfuncitem.funcargs.get("uvloop_loop")
or pyfuncitem.funcargs.get("loop", None)
)
with _runtime_warning_context():
with _passthrough_loop_context(existing_loop, fast=fast) as _loop:
testargs = {
arg: pyfuncitem.funcargs[arg]
for arg in pyfuncitem._fixtureinfo.argnames
}
_loop.run_until_complete(pyfuncitem.obj(**testargs))
return True
def pytest_generate_tests(metafunc): # type: ignore[no-untyped-def]
if "loop_factory" not in metafunc.fixturenames:
return
loops = metafunc.config.option.aiohttp_loop
avail_factories: dict[str, Callable[[], asyncio.AbstractEventLoop]]
avail_factories = {"pyloop": asyncio.new_event_loop}
if uvloop is not None: # pragma: no cover
avail_factories["uvloop"] = uvloop.new_event_loop
if loops == "all":
loops = "pyloop,uvloop?"
factories = {} # type: ignore[var-annotated]
for name in loops.split(","):
required = not name.endswith("?")
name = name.strip(" ?")
if name not in avail_factories: # pragma: no cover
if required:
raise ValueError(
"Unknown loop '%s', available loops: %s"
% (name, list(factories.keys()))
)
else:
continue
factories[name] = avail_factories[name]
metafunc.parametrize(
"loop_factory", list(factories.values()), ids=list(factories.keys())
)
@pytest.fixture
def loop(
loop_factory: Callable[[], asyncio.AbstractEventLoop],
fast: bool,
loop_debug: bool,
) -> Iterator[asyncio.AbstractEventLoop]:
"""Return an instance of the event loop."""
with loop_context(loop_factory, fast=fast) as _loop:
if loop_debug:
_loop.set_debug(True) # pragma: no cover
asyncio.set_event_loop(_loop)
yield _loop
@pytest.fixture
def proactor_loop() -> Iterator[asyncio.AbstractEventLoop]:
factory = asyncio.ProactorEventLoop # type: ignore[attr-defined]
with loop_context(factory) as _loop:
asyncio.set_event_loop(_loop)
yield _loop
@pytest.fixture
def unused_port(aiohttp_unused_port: Callable[[], int]) -> Callable[[], int]:
warnings.warn(
"Deprecated, use aiohttp_unused_port fixture instead",
DeprecationWarning,
stacklevel=2,
)
return aiohttp_unused_port
@pytest.fixture
def aiohttp_unused_port() -> Callable[[], int]:
"""Return a port that is unused on the current host."""
return _unused_port
@pytest.fixture
def aiohttp_server(loop: asyncio.AbstractEventLoop) -> Iterator[AiohttpServer]:
"""Factory to create a TestServer instance, given an app.
aiohttp_server(app, **kwargs)
"""
servers = []
async def go(
app: Application,
*,
host: str = "127.0.0.1",
port: int | None = None,
**kwargs: Any,
) -> TestServer:
server = TestServer(app, host=host, port=port)
await server.start_server(loop=loop, **kwargs)
servers.append(server)
return server
yield go
async def finalize() -> None:
while servers:
await servers.pop().close()
loop.run_until_complete(finalize())
@pytest.fixture
def test_server(aiohttp_server): # type: ignore[no-untyped-def] # pragma: no cover
warnings.warn(
"Deprecated, use aiohttp_server fixture instead",
DeprecationWarning,
stacklevel=2,
)
return aiohttp_server
@pytest.fixture
def aiohttp_raw_server(loop: asyncio.AbstractEventLoop) -> Iterator[AiohttpRawServer]:
"""Factory to create a RawTestServer instance, given a web handler.
aiohttp_raw_server(handler, **kwargs)
"""
servers = []
async def go(
handler: _RequestHandler, *, port: int | None = None, **kwargs: Any
) -> RawTestServer:
server = RawTestServer(handler, port=port)
await server.start_server(loop=loop, **kwargs)
servers.append(server)
return server
yield go
async def finalize() -> None:
while servers:
await servers.pop().close()
loop.run_until_complete(finalize())
@pytest.fixture
def raw_test_server( # type: ignore[no-untyped-def] # pragma: no cover
aiohttp_raw_server,
):
warnings.warn(
"Deprecated, use aiohttp_raw_server fixture instead",
DeprecationWarning,
stacklevel=2,
)
return aiohttp_raw_server
@pytest.fixture
def aiohttp_client(loop: asyncio.AbstractEventLoop) -> Iterator[AiohttpClient]:
"""Factory to create a TestClient instance.
aiohttp_client(app, **kwargs)
aiohttp_client(server, **kwargs)
aiohttp_client(raw_server, **kwargs)
"""
clients = []
@overload
async def go(
__param: Application,
*,
server_kwargs: dict[str, Any] | None = None,
**kwargs: Any,
) -> TestClient[Request, Application]: ...
@overload
async def go(
__param: BaseTestServer,
*,
server_kwargs: dict[str, Any] | None = None,
**kwargs: Any,
) -> TestClient[BaseRequest, None]: ...
async def go(
__param: Application | BaseTestServer,
*args: Any,
server_kwargs: dict[str, Any] | None = None,
**kwargs: Any,
) -> TestClient[Any, Any]:
if isinstance(__param, Callable) and not isinstance( # type: ignore[arg-type]
__param, (Application, BaseTestServer)
):
__param = __param(loop, *args, **kwargs)
kwargs = {}
else:
assert not args, "args should be empty"
if isinstance(__param, Application):
server_kwargs = server_kwargs or {}
server = TestServer(__param, loop=loop, **server_kwargs)
client = TestClient(server, loop=loop, **kwargs)
elif isinstance(__param, BaseTestServer):
client = TestClient(__param, loop=loop, **kwargs)
else:
raise ValueError("Unknown argument type: %r" % type(__param))
await client.start_server()
clients.append(client)
return client
yield go
async def finalize() -> None:
while clients:
await clients.pop().close()
loop.run_until_complete(finalize())
@pytest.fixture
def test_client(aiohttp_client): # type: ignore[no-untyped-def] # pragma: no cover
warnings.warn(
"Deprecated, use aiohttp_client fixture instead",
DeprecationWarning,
stacklevel=2,
)
return aiohttp_client
+301
View File
@@ -0,0 +1,301 @@
import asyncio
import socket
import sys
import weakref
from typing import Any, Final, Optional
from .abc import AbstractResolver, ResolveResult
__all__ = ("ThreadedResolver", "AsyncResolver", "DefaultResolver")
try:
import aiodns
aiodns_default = hasattr(aiodns.DNSResolver, "getaddrinfo")
except ImportError: # pragma: no cover
aiodns = None # type: ignore[assignment]
aiodns_default = False
_NUMERIC_SOCKET_FLAGS = socket.AI_NUMERICHOST | socket.AI_NUMERICSERV
_NAME_SOCKET_FLAGS = socket.NI_NUMERICHOST | socket.NI_NUMERICSERV
_AI_ADDRCONFIG = socket.AI_ADDRCONFIG
if hasattr(socket, "AI_MASK"):
_AI_ADDRCONFIG &= socket.AI_MASK
_IS_WINDOWS = sys.platform == "win32"
def _is_windows_localhost(host: str) -> bool:
return _IS_WINDOWS and host.rstrip(".").casefold() == "localhost"
class ThreadedResolver(AbstractResolver):
"""Threaded resolver.
Uses an Executor for synchronous getaddrinfo() calls.
concurrent.futures.ThreadPoolExecutor is used by default.
"""
def __init__(self, loop: asyncio.AbstractEventLoop | None = None) -> None:
self._loop = loop or asyncio.get_running_loop()
async def resolve(
self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET
) -> list[ResolveResult]:
try:
infos = await self._loop.getaddrinfo(
host,
port,
type=socket.SOCK_STREAM,
family=family,
flags=_AI_ADDRCONFIG,
)
except socket.gaierror:
if not _is_windows_localhost(host):
raise
infos = await self._loop.getaddrinfo(
host,
port,
type=socket.SOCK_STREAM,
family=family,
flags=0,
)
hosts: list[ResolveResult] = []
for family, _, proto, _, address in infos:
if family == socket.AF_INET6:
if len(address) < 3:
# IPv6 is not supported by Python build,
# or IPv6 is not enabled in the host
continue
if address[3]:
# This is essential for link-local IPv6 addresses.
# LL IPv6 is a VERY rare case. Strictly speaking, we should use
# getnameinfo() unconditionally, but performance makes sense.
resolved_host, _port = await self._loop.getnameinfo(
address, _NAME_SOCKET_FLAGS
)
port = int(_port)
else:
resolved_host, port = address[:2]
else: # IPv4
assert family == socket.AF_INET
resolved_host, port = address # type: ignore[misc]
hosts.append(
ResolveResult(
hostname=host,
host=resolved_host,
port=port,
family=family,
proto=proto,
flags=_NUMERIC_SOCKET_FLAGS,
)
)
return hosts
async def close(self) -> None:
pass
class AsyncResolver(AbstractResolver):
"""Use the `aiodns` package to make asynchronous DNS lookups"""
def __init__(
self,
loop: asyncio.AbstractEventLoop | None = None,
*args: Any,
**kwargs: Any,
) -> None:
if aiodns is None:
raise RuntimeError("Resolver requires aiodns library")
self._loop = loop or asyncio.get_running_loop()
self._manager: _DNSResolverManager | None = None
# If custom args are provided, create a dedicated resolver instance
# This means each AsyncResolver with custom args gets its own
# aiodns.DNSResolver instance
if args or kwargs:
self._resolver = aiodns.DNSResolver(*args, **kwargs)
return
# Use the shared resolver from the manager for default arguments
self._manager = _DNSResolverManager()
self._resolver = self._manager.get_resolver(self, self._loop)
if not hasattr(self._resolver, "gethostbyname"):
# aiodns 1.1 is not available, fallback to DNSResolver.query
self.resolve = self._resolve_with_query # type: ignore
async def resolve(
self, host: str, port: int = 0, family: socket.AddressFamily = socket.AF_INET
) -> list[ResolveResult]:
try:
try:
resp = await self._resolver.getaddrinfo(
host,
port=port,
type=socket.SOCK_STREAM,
family=family,
flags=_AI_ADDRCONFIG,
)
except aiodns.error.DNSError:
if not _is_windows_localhost(host):
raise
resp = await self._resolver.getaddrinfo(
host,
port=port,
type=socket.SOCK_STREAM,
family=family,
flags=0,
)
except aiodns.error.DNSError as exc:
msg = exc.args[1] if len(exc.args) >= 1 else "DNS lookup failed"
raise OSError(None, msg) from exc
hosts: list[ResolveResult] = []
for node in resp.nodes:
address: tuple[bytes, int] | tuple[bytes, int, int, int] = node.addr
if node.family == socket.AF_INET6:
if len(address) > 3 and address[3]:
# This is essential for link-local IPv6 addresses.
# LL IPv6 is a VERY rare case. Strictly speaking, we should use
# getnameinfo() unconditionally, but performance makes sense.
result = await self._resolver.getnameinfo(
(address[0].decode("ascii"), *address[1:]),
_NAME_SOCKET_FLAGS,
)
resolved_host = result.node
else:
resolved_host = address[0].decode("ascii")
port = address[1]
else: # IPv4
assert node.family == socket.AF_INET
resolved_host = address[0].decode("ascii")
port = address[1]
hosts.append(
ResolveResult(
hostname=host,
host=resolved_host,
port=port,
family=node.family,
proto=0,
flags=_NUMERIC_SOCKET_FLAGS,
)
)
if not hosts:
raise OSError(None, "DNS lookup failed")
return hosts
async def _resolve_with_query(
self, host: str, port: int = 0, family: int = socket.AF_INET
) -> list[dict[str, Any]]:
qtype: Final = "AAAA" if family == socket.AF_INET6 else "A"
try:
resp = await self._resolver.query(host, qtype)
except aiodns.error.DNSError as exc:
msg = exc.args[1] if len(exc.args) >= 1 else "DNS lookup failed"
raise OSError(None, msg) from exc
hosts = []
for rr in resp:
hosts.append(
{
"hostname": host,
"host": rr.host,
"port": port,
"family": family,
"proto": 0,
"flags": socket.AI_NUMERICHOST,
}
)
if not hosts:
raise OSError(None, "DNS lookup failed")
return hosts
async def close(self) -> None:
if self._manager:
# Release the resolver from the manager if using the shared resolver
self._manager.release_resolver(self, self._loop)
self._manager = None # Clear reference to manager
self._resolver = None # type: ignore[assignment] # Clear reference to resolver
return
# Otherwise cancel our dedicated resolver
if self._resolver is not None:
self._resolver.cancel()
self._resolver = None # type: ignore[assignment] # Clear reference
class _DNSResolverManager:
"""Manager for aiodns.DNSResolver objects.
This class manages shared aiodns.DNSResolver instances
with no custom arguments across different event loops.
"""
_instance: Optional["_DNSResolverManager"] = None
def __new__(cls) -> "_DNSResolverManager":
if cls._instance is None:
cls._instance = super().__new__(cls)
cls._instance._init()
return cls._instance
def _init(self) -> None:
# Use WeakKeyDictionary to allow event loops to be garbage collected
self._loop_data: weakref.WeakKeyDictionary[
asyncio.AbstractEventLoop,
tuple[aiodns.DNSResolver, weakref.WeakSet[AsyncResolver]],
] = weakref.WeakKeyDictionary()
def get_resolver(
self, client: "AsyncResolver", loop: asyncio.AbstractEventLoop
) -> "aiodns.DNSResolver":
"""Get or create the shared aiodns.DNSResolver instance for a specific event loop.
Args:
client: The AsyncResolver instance requesting the resolver.
This is required to track resolver usage.
loop: The event loop to use for the resolver.
"""
# Create a new resolver and client set for this loop if it doesn't exist
if loop not in self._loop_data:
resolver = aiodns.DNSResolver(loop=loop)
client_set: weakref.WeakSet[AsyncResolver] = weakref.WeakSet()
self._loop_data[loop] = (resolver, client_set)
else:
# Get the existing resolver and client set
resolver, client_set = self._loop_data[loop]
# Register this client with the loop
client_set.add(client)
return resolver
def release_resolver(
self, client: "AsyncResolver", loop: asyncio.AbstractEventLoop
) -> None:
"""Release the resolver for an AsyncResolver client when it's closed.
Args:
client: The AsyncResolver instance to release.
loop: The event loop the resolver was using.
"""
# Remove client from its loop's tracking
current_loop_data = self._loop_data.get(loop)
if current_loop_data is None:
return
resolver, client_set = current_loop_data
client_set.discard(client)
# If no more clients for this loop, cancel and remove its resolver
if not client_set:
if resolver is not None:
resolver.cancel()
del self._loop_data[loop]
_DefaultType = type[AsyncResolver | ThreadedResolver]
DefaultResolver: _DefaultType = AsyncResolver if aiodns_default else ThreadedResolver
+760
View File
@@ -0,0 +1,760 @@
import asyncio
import collections
import sys
import warnings
from collections.abc import Awaitable, Callable
from typing import Final, Generic, TypeVar
from .base_protocol import BaseProtocol
from .helpers import (
_EXC_SENTINEL,
DEFAULT_CHUNK_SIZE,
BaseTimerContext,
TimerNoop,
set_exception,
set_result,
)
from .http_exceptions import LineTooLong
from .log import internal_logger
__all__ = (
"EMPTY_PAYLOAD",
"EofStream",
"StreamReader",
"DataQueue",
)
_T = TypeVar("_T")
class EofStream(Exception):
"""eof stream indication."""
class AsyncStreamIterator(Generic[_T]):
__slots__ = ("read_func",)
def __init__(self, read_func: Callable[[], Awaitable[_T]]) -> None:
self.read_func = read_func
def __aiter__(self) -> "AsyncStreamIterator[_T]":
return self
async def __anext__(self) -> _T:
try:
rv = await self.read_func()
except EofStream:
raise StopAsyncIteration
if rv == b"":
raise StopAsyncIteration
return rv
class ChunkTupleAsyncStreamIterator:
__slots__ = ("_stream",)
def __init__(self, stream: "StreamReader") -> None:
self._stream = stream
def __aiter__(self) -> "ChunkTupleAsyncStreamIterator":
return self
async def __anext__(self) -> tuple[bytes, bool]:
rv = await self._stream.readchunk()
if rv == (b"", False):
raise StopAsyncIteration
return rv
class StreamReader:
"""An enhancement of asyncio.StreamReader.
Supports asynchronous iteration by line, chunk or as available::
async for line in reader:
...
async for chunk in reader.iter_chunked(1024):
...
async for slice in reader.iter_any():
...
"""
__slots__ = (
"_protocol",
"_low_water",
"_high_water",
"_low_water_chunks",
"_high_water_chunks",
"_loop",
"_size",
"_cursor",
"_http_chunk_splits",
"_buffer",
"_buffer_offset",
"_eof",
"_waiter",
"_eof_waiter",
"_exception",
"_timer",
"_eof_callbacks",
"_eof_counter",
"total_bytes",
"total_compressed_bytes",
)
def __init__(
self,
protocol: BaseProtocol,
limit: int,
*,
timer: BaseTimerContext | None = None,
loop: asyncio.AbstractEventLoop | None = None,
) -> None:
self._protocol = protocol
self._low_water = limit
self._high_water = limit * 2
if loop is None:
loop = asyncio.get_event_loop()
# Use max(4, ...) because there's always at least 1 chunk split remaining
# (the current position), so we need low_water >= 2 to allow resume.
# limit // 16 gets us a reasonable value of 16k with default 256KiB limit.
self._high_water_chunks = max(4, limit // 16)
self._low_water_chunks = self._high_water_chunks // 2
self._loop = loop
self._size = 0
self._cursor = 0
self._http_chunk_splits: collections.deque[int] | None = None
self._buffer: collections.deque[bytes] = collections.deque()
self._buffer_offset = 0
self._eof = False
self._waiter: asyncio.Future[None] | None = None
self._eof_waiter: asyncio.Future[None] | None = None
self._exception: BaseException | None = None
self._timer = TimerNoop() if timer is None else timer
self._eof_callbacks: list[Callable[[], None]] = []
self._eof_counter = 0
self.total_bytes = 0
self.total_compressed_bytes: int | None = None
def __repr__(self) -> str:
info = [self.__class__.__name__]
if self._size:
info.append("%d bytes" % self._size)
if self._eof:
info.append("eof")
if self._low_water != DEFAULT_CHUNK_SIZE:
info.append("low=%d high=%d" % (self._low_water, self._high_water))
if self._waiter:
info.append("w=%r" % self._waiter)
if self._exception:
info.append("e=%r" % self._exception)
return "<%s>" % " ".join(info)
def __aiter__(self) -> AsyncStreamIterator[bytes]:
return AsyncStreamIterator(self.readline)
def iter_chunked(self, n: int) -> AsyncStreamIterator[bytes]:
"""Returns an asynchronous iterator that yields chunks of size n."""
self.set_read_chunk_size(n)
return AsyncStreamIterator(lambda: self.read(n))
def iter_any(self) -> AsyncStreamIterator[bytes]:
"""Yield all available data as soon as it is received."""
return AsyncStreamIterator(self.readany)
def iter_chunks(self) -> ChunkTupleAsyncStreamIterator:
"""Yield chunks of data as they are received by the server.
The yielded objects are tuples
of (bytes, bool) as returned by the StreamReader.readchunk method.
"""
return ChunkTupleAsyncStreamIterator(self)
def get_read_buffer_limits(self) -> tuple[int, int]:
return (self._low_water, self._high_water)
def set_read_chunk_size(self, n: int) -> None:
"""Raise buffer limits to match the consumer's chunk size."""
if n > self._low_water:
self._low_water = n
self._high_water = n * 2
def exception(self) -> BaseException | None:
return self._exception
def set_exception(
self,
exc: BaseException,
exc_cause: BaseException = _EXC_SENTINEL,
) -> None:
self._exception = exc
self._eof_callbacks.clear()
waiter = self._waiter
if waiter is not None:
self._waiter = None
set_exception(waiter, exc, exc_cause)
waiter = self._eof_waiter
if waiter is not None:
self._eof_waiter = None
set_exception(waiter, exc, exc_cause)
def on_eof(self, callback: Callable[[], None]) -> None:
if self._eof:
try:
callback()
except Exception:
internal_logger.exception("Exception in eof callback")
else:
self._eof_callbacks.append(callback)
def feed_eof(self) -> None:
self._eof = True
waiter = self._waiter
if waiter is not None:
self._waiter = None
set_result(waiter, None)
waiter = self._eof_waiter
if waiter is not None:
self._eof_waiter = None
set_result(waiter, None)
# At EOF the parser is done, there won't be unprocessed data.
self._protocol.resume_reading(resume_parser=False)
for cb in self._eof_callbacks:
try:
cb()
except Exception:
internal_logger.exception("Exception in eof callback")
self._eof_callbacks.clear()
def is_eof(self) -> bool:
"""Return True if 'feed_eof' was called."""
return self._eof
def at_eof(self) -> bool:
"""Return True if the buffer is empty and 'feed_eof' was called."""
return self._eof and not self._buffer
async def wait_eof(self) -> None:
if self._eof:
return
assert self._eof_waiter is None
self._eof_waiter = self._loop.create_future()
try:
await self._eof_waiter
finally:
self._eof_waiter = None
@property
def total_raw_bytes(self) -> int:
if self.total_compressed_bytes is None:
return self.total_bytes
return self.total_compressed_bytes
def unread_data(self, data: bytes) -> None:
"""rollback reading some data from stream, inserting it to buffer head."""
warnings.warn(
"unread_data() is deprecated "
"and will be removed in future releases (#3260)",
DeprecationWarning,
stacklevel=2,
)
if not data:
return
if self._buffer_offset:
self._buffer[0] = self._buffer[0][self._buffer_offset :]
self._buffer_offset = 0
self._size += len(data)
self._cursor -= len(data)
self._buffer.appendleft(data)
self._eof_counter = 0
# TODO: size is ignored, remove the param later
def feed_data(self, data: bytes, size: int = 0) -> bool:
assert not self._eof, "feed_data after feed_eof"
if not data:
return False
data_len = len(data)
self._size += data_len
self._buffer.append(data)
self.total_bytes += data_len
waiter = self._waiter
if waiter is not None:
self._waiter = None
set_result(waiter, None)
if self._size > self._high_water:
self._protocol.pause_reading()
return False
def begin_http_chunk_receiving(self) -> None:
if self._http_chunk_splits is None:
if self.total_bytes:
raise RuntimeError(
"Called begin_http_chunk_receiving when some data was already fed"
)
self._http_chunk_splits = collections.deque()
def end_http_chunk_receiving(self) -> None:
if self._http_chunk_splits is None:
raise RuntimeError(
"Called end_chunk_receiving without calling "
"begin_chunk_receiving first"
)
# self._http_chunk_splits contains logical byte offsets from start of
# the body transfer. Each offset is the offset of the end of a chunk.
# "Logical" means bytes, accessible for a user.
# If no chunks containing logical data were received, current position
# is difinitely zero.
pos = self._http_chunk_splits[-1] if self._http_chunk_splits else 0
if self.total_bytes == pos:
# We should not add empty chunks here. So we check for that.
# Note, when chunked + gzip is used, we can receive a chunk
# of compressed data, but that data may not be enough for gzip FSM
# to yield any uncompressed data. That's why current position may
# not change after receiving a chunk.
return
self._http_chunk_splits.append(self.total_bytes)
# If we get too many small chunks before self._high_water is reached, then any
# .read() call becomes computationally expensive, and could block the event loop
# for too long, hence an additional self._high_water_chunks here.
if len(self._http_chunk_splits) > self._high_water_chunks:
self._protocol.pause_reading()
# wake up readchunk when end of http chunk received
waiter = self._waiter
if waiter is not None:
self._waiter = None
set_result(waiter, None)
async def _wait(self, func_name: str) -> None:
if not self._protocol.connected:
raise RuntimeError("Connection closed.")
# StreamReader uses a future to link the protocol feed_data() method
# to a read coroutine. Running two read coroutines at the same time
# would have an unexpected behaviour. It would not possible to know
# which coroutine would get the next data.
if self._waiter is not None:
raise RuntimeError(
"%s() called while another coroutine is "
"already waiting for incoming data" % func_name
)
waiter = self._waiter = self._loop.create_future()
try:
with self._timer:
await waiter
finally:
self._waiter = None
async def readline(self, *, max_line_length: int | None = None) -> bytes:
return await self.readuntil(max_size=max_line_length)
async def readuntil(
self, separator: bytes = b"\n", *, max_size: int | None = None
) -> bytes:
seplen = len(separator)
if seplen == 0:
raise ValueError("Separator should be at least one-byte string")
if self._exception is not None:
raise self._exception
chunk = b""
chunk_size = 0
not_enough = True
max_size = max_size or self._high_water
while not_enough:
while self._buffer and not_enough:
offset = self._buffer_offset
ichar = self._buffer[0].find(separator, offset) + 1
# Read from current offset to found separator or to the end.
data = self._read_nowait_chunk(
ichar - offset + seplen - 1 if ichar else -1
)
chunk += data
chunk_size += len(data)
if ichar:
not_enough = False
if chunk_size > max_size:
raise LineTooLong(chunk[:100] + b"...", max_size)
if self._eof:
break
if not_enough:
await self._wait("readuntil")
return chunk
async def read(self, n: int = -1) -> bytes:
if self._exception is not None:
raise self._exception
# migration problem; with DataQueue you have to catch
# EofStream exception, so common way is to run payload.read() inside
# infinite loop. what can cause real infinite loop with StreamReader
# lets keep this code one major release.
if __debug__:
if self._eof and not self._buffer:
self._eof_counter = getattr(self, "_eof_counter", 0) + 1
if self._eof_counter > 5:
internal_logger.warning(
"Multiple access to StreamReader in eof state, "
"might be infinite loop.",
stack_info=True,
)
if not n:
return b""
if n < 0:
# Reading everything — remove decompression chunk limit.
self.set_read_chunk_size(sys.maxsize)
blocks = []
while True:
block = await self.readany()
if not block:
break
blocks.append(block)
return b"".join(blocks)
self.set_read_chunk_size(n)
# TODO: should be `if` instead of `while`
# because waiter maybe triggered on chunk end,
# without feeding any data
while not self._buffer and not self._eof:
await self._wait("read")
return self._read_nowait(n)
async def readany(self) -> bytes:
if self._exception is not None:
raise self._exception
# TODO: should be `if` instead of `while`
# because waiter maybe triggered on chunk end,
# without feeding any data
while not self._buffer and not self._eof:
await self._wait("readany")
return self._read_nowait(-1)
async def readchunk(self) -> tuple[bytes, bool]:
"""Returns a tuple of (data, end_of_http_chunk).
When chunked transfer
encoding is used, end_of_http_chunk is a boolean indicating if the end
of the data corresponds to the end of a HTTP chunk , otherwise it is
always False.
"""
while True:
if self._exception is not None:
raise self._exception
while self._http_chunk_splits:
pos = self._http_chunk_splits.popleft()
if pos == self._cursor:
return (b"", True)
if pos > self._cursor:
return (self._read_nowait(pos - self._cursor), True)
internal_logger.warning(
"Skipping HTTP chunk end due to data "
"consumption beyond chunk boundary"
)
if self._buffer:
return (self._read_nowait_chunk(-1), False)
# return (self._read_nowait(-1), False)
if self._eof:
# Special case for signifying EOF.
# (b'', True) is not a final return value actually.
return (b"", False)
await self._wait("readchunk")
async def readexactly(self, n: int) -> bytes:
if self._exception is not None:
raise self._exception
blocks: list[bytes] = []
while n > 0:
block = await self.read(n)
if not block:
partial = b"".join(blocks)
raise asyncio.IncompleteReadError(partial, len(partial) + n)
blocks.append(block)
n -= len(block)
return b"".join(blocks)
def read_nowait(self, n: int = -1) -> bytes:
# default was changed to be consistent with .read(-1)
#
# I believe the most users don't know about the method and
# they are not affected.
if self._exception is not None:
raise self._exception
if self._waiter and not self._waiter.done():
raise RuntimeError(
"Called while some coroutine is waiting for incoming data."
)
return self._read_nowait(n)
def _read_nowait_chunk(self, n: int) -> bytes:
first_buffer = self._buffer[0]
offset = self._buffer_offset
if n != -1 and len(first_buffer) - offset > n:
data = first_buffer[offset : offset + n]
self._buffer_offset += n
elif offset:
self._buffer.popleft()
data = first_buffer[offset:]
self._buffer_offset = 0
else:
data = self._buffer.popleft()
data_len = len(data)
self._size -= data_len
self._cursor += data_len
chunk_splits = self._http_chunk_splits
# Prevent memory leak: drop useless chunk splits
while chunk_splits and chunk_splits[0] < self._cursor:
chunk_splits.popleft()
if self._size < self._low_water and (
self._http_chunk_splits is None
or len(self._http_chunk_splits) < self._low_water_chunks
):
self._protocol.resume_reading()
return data
def _read_nowait(self, n: int) -> bytes:
"""Read not more than n bytes, or whole buffer if n == -1"""
self._timer.assert_timeout()
if n == -1:
# Drain only chunks present now; _read_nowait_chunk() can
# re-entrantly resume_reading() and refill the buffer.
count = len(self._buffer)
if count == 1:
return self._read_nowait_chunk(-1)
return b"".join([self._read_nowait_chunk(-1) for _ in range(count)])
chunks: list[bytes] = []
while self._buffer:
chunk = self._read_nowait_chunk(n)
chunks.append(chunk)
n -= len(chunk)
if n == 0:
break
return b"".join(chunks) if chunks else b""
class EmptyStreamReader(StreamReader): # lgtm [py/missing-call-to-init]
__slots__ = ("_read_eof_chunk",)
def __init__(self) -> None:
self._read_eof_chunk = False
self.total_bytes = 0
def __repr__(self) -> str:
return "<%s>" % self.__class__.__name__
def exception(self) -> BaseException | None:
return None
def set_exception(
self,
exc: BaseException,
exc_cause: BaseException = _EXC_SENTINEL,
) -> None:
pass
def on_eof(self, callback: Callable[[], None]) -> None:
try:
callback()
except Exception:
internal_logger.exception("Exception in eof callback")
def feed_eof(self) -> None:
pass
def is_eof(self) -> bool:
return True
def at_eof(self) -> bool:
return True
async def wait_eof(self) -> None:
return
def feed_data(self, data: bytes, n: int = 0) -> bool:
return False
def set_read_chunk_size(self, n: int) -> None:
return
async def readline(self, *, max_line_length: int | None = None) -> bytes:
return b""
async def read(self, n: int = -1) -> bytes:
return b""
# TODO add async def readuntil
async def readany(self) -> bytes:
return b""
async def readchunk(self) -> tuple[bytes, bool]:
if not self._read_eof_chunk:
self._read_eof_chunk = True
return (b"", False)
return (b"", True)
async def readexactly(self, n: int) -> bytes:
raise asyncio.IncompleteReadError(b"", n)
def read_nowait(self, n: int = -1) -> bytes:
return b""
EMPTY_PAYLOAD: Final[StreamReader] = EmptyStreamReader()
class DataQueue(Generic[_T]):
"""DataQueue is a general-purpose blocking queue with one reader."""
def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
self._loop = loop
self._eof = False
self._waiter: asyncio.Future[None] | None = None
self._exception: BaseException | None = None
self._buffer: collections.deque[tuple[_T, int]] = collections.deque()
def __len__(self) -> int:
return len(self._buffer)
def is_eof(self) -> bool:
return self._eof
def at_eof(self) -> bool:
return self._eof and not self._buffer
def exception(self) -> BaseException | None:
return self._exception
def set_exception(
self,
exc: BaseException,
exc_cause: BaseException = _EXC_SENTINEL,
) -> None:
self._eof = True
self._exception = exc
if (waiter := self._waiter) is not None:
self._waiter = None
set_exception(waiter, exc, exc_cause)
def feed_data(self, data: _T, size: int = 0) -> None:
self._buffer.append((data, size))
if (waiter := self._waiter) is not None:
self._waiter = None
set_result(waiter, None)
def feed_eof(self) -> None:
self._eof = True
if (waiter := self._waiter) is not None:
self._waiter = None
set_result(waiter, None)
async def read(self) -> _T:
if not self._buffer and not self._eof:
assert not self._waiter
self._waiter = self._loop.create_future()
try:
await self._waiter
except (asyncio.CancelledError, asyncio.TimeoutError):
self._waiter = None
raise
if self._buffer:
data, _ = self._buffer.popleft()
return data
if self._exception is not None:
raise self._exception
raise EofStream
def __aiter__(self) -> AsyncStreamIterator[_T]:
return AsyncStreamIterator(self.read)
class FlowControlDataQueue(DataQueue[_T]):
"""FlowControlDataQueue resumes and pauses an underlying stream.
It is a destination for parsed data.
This class is deprecated and will be removed in version 4.0.
"""
def __init__(
self, protocol: BaseProtocol, limit: int, *, loop: asyncio.AbstractEventLoop
) -> None:
super().__init__(loop=loop)
self._size = 0
self._protocol = protocol
self._limit = limit * 2
def feed_data(self, data: _T, size: int = 0) -> None:
super().feed_data(data, size)
self._size += size
if self._size > self._limit and not self._protocol._reading_paused:
self._protocol.pause_reading()
async def read(self) -> _T:
if not self._buffer and not self._eof:
assert not self._waiter
self._waiter = self._loop.create_future()
try:
await self._waiter
except (asyncio.CancelledError, asyncio.TimeoutError):
self._waiter = None
raise
if self._buffer:
data, size = self._buffer.popleft()
self._size -= size
if self._size < self._limit and self._protocol._reading_paused:
self._protocol.resume_reading()
return data
if self._exception is not None:
raise self._exception
raise EofStream
+37
View File
@@ -0,0 +1,37 @@
"""Helper methods to tune a TCP connection"""
import asyncio
import socket
from contextlib import suppress
from typing import Optional # noqa
__all__ = ("tcp_keepalive", "tcp_nodelay")
if hasattr(socket, "SO_KEEPALIVE"):
def tcp_keepalive(transport: asyncio.Transport) -> None:
sock = transport.get_extra_info("socket")
if sock is not None:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
else:
def tcp_keepalive(transport: asyncio.Transport) -> None: # pragma: no cover
pass
def tcp_nodelay(transport: asyncio.Transport, value: bool) -> None:
sock = transport.get_extra_info("socket")
if sock is None:
return
if sock.family not in (socket.AF_INET, socket.AF_INET6):
return
value = bool(value)
# socket may be closed already, on windows OSError get raised
with suppress(OSError):
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, value)
+808
View File
@@ -0,0 +1,808 @@
"""Utilities shared by tests."""
import asyncio
import contextlib
import gc
import inspect
import ipaddress
import os
import socket
import sys
import warnings
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterator
from types import TracebackType
from typing import TYPE_CHECKING, Any, Generic, Literal, TypeVar, cast, overload
from unittest import IsolatedAsyncioTestCase, mock
from aiosignal import Signal
from multidict import CIMultiDict, CIMultiDictProxy
from yarl import URL
import aiohttp
from aiohttp.client import (
_BaseRequestContextManager,
_RequestContextManager,
_RequestOptions,
_WSRequestContextManager,
)
from . import ClientSession, hdrs
from .abc import AbstractCookieJar
from .client_reqrep import ClientResponse
from .client_ws import ClientWebSocketResponse
from .helpers import sentinel
from .http import HttpVersion, RawRequestMessage
from .streams import EMPTY_PAYLOAD, StreamReader
from .typedefs import StrOrURL
from .web import (
Application,
AppRunner,
BaseRequest,
BaseRunner,
Request,
Server,
ServerRunner,
SockSite,
UrlMappingMatchInfo,
)
from .web_protocol import _RequestHandler
if TYPE_CHECKING:
from ssl import SSLContext
else:
SSLContext = Any
if sys.version_info >= (3, 11) and TYPE_CHECKING:
from typing import Unpack
if sys.version_info >= (3, 11):
from typing import Self
else:
Self = Any
_ApplicationNone = TypeVar("_ApplicationNone", Application, None)
_Request = TypeVar("_Request", bound=BaseRequest)
REUSE_ADDRESS = os.name == "posix" and sys.platform != "cygwin"
def get_unused_port_socket(
host: str, family: socket.AddressFamily = socket.AF_INET
) -> socket.socket:
return get_port_socket(host, 0, family)
def get_port_socket(
host: str, port: int, family: socket.AddressFamily
) -> socket.socket:
s = socket.socket(family, socket.SOCK_STREAM)
if REUSE_ADDRESS:
# Windows has different semantics for SO_REUSEADDR,
# so don't set it. Ref:
# https://docs.microsoft.com/en-us/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host, port))
return s
def unused_port() -> int:
"""Return a port that is unused on the current host."""
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("127.0.0.1", 0))
return cast(int, s.getsockname()[1])
class BaseTestServer(ABC):
__test__ = False
def __init__(
self,
*,
scheme: str = "",
loop: asyncio.AbstractEventLoop | None = None,
host: str = "127.0.0.1",
port: int | None = None,
skip_url_asserts: bool = False,
socket_factory: Callable[
[str, int, socket.AddressFamily], socket.socket
] = get_port_socket,
**kwargs: Any,
) -> None:
self._loop = loop
self.runner: BaseRunner | None = None
self._root: URL | None = None
self.host = host
self.port = port
self._closed = False
self.scheme = scheme
self.skip_url_asserts = skip_url_asserts
self.socket_factory = socket_factory
async def start_server(
self, loop: asyncio.AbstractEventLoop | None = None, **kwargs: Any
) -> None:
if self.runner:
return
self._loop = loop
self._ssl = kwargs.pop("ssl", None)
self.runner = await self._make_runner(handler_cancellation=True, **kwargs)
await self.runner.setup()
if not self.port:
self.port = 0
absolute_host = self.host
try:
version = ipaddress.ip_address(self.host).version
except ValueError:
version = 4
if version == 6:
absolute_host = f"[{self.host}]"
family = socket.AF_INET6 if version == 6 else socket.AF_INET
_sock = self.socket_factory(self.host, self.port, family)
self.host, self.port = _sock.getsockname()[:2]
site = SockSite(self.runner, sock=_sock, ssl_context=self._ssl)
await site.start()
server = site._server
assert server is not None
sockets = server.sockets # type: ignore[attr-defined]
assert sockets is not None
self.port = sockets[0].getsockname()[1]
if not self.scheme:
self.scheme = "https" if self._ssl else "http"
self._root = URL(f"{self.scheme}://{absolute_host}:{self.port}")
@abstractmethod # pragma: no cover
async def _make_runner(self, **kwargs: Any) -> BaseRunner:
pass
def make_url(self, path: StrOrURL) -> URL:
assert self._root is not None
url = URL(path)
if not self.skip_url_asserts:
assert not url.absolute
return self._root.join(url)
else:
return URL(str(self._root) + str(path))
@property
def started(self) -> bool:
return self.runner is not None
@property
def closed(self) -> bool:
return self._closed
@property
def handler(self) -> Server:
# for backward compatibility
# web.Server instance
runner = self.runner
assert runner is not None
assert runner.server is not None
return runner.server
async def close(self) -> None:
"""Close all fixtures created by the test client.
After that point, the TestClient is no longer usable.
This is an idempotent function: running close multiple times
will not have any additional effects.
close is also run when the object is garbage collected, and on
exit when used as a context manager.
"""
if self.started and not self.closed:
assert self.runner is not None
await self.runner.cleanup()
self._root = None
self.port = None
self._closed = True
def __enter__(self) -> None:
raise TypeError("Use async with instead")
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
# __exit__ should exist in pair with __enter__ but never executed
pass # pragma: no cover
async def __aenter__(self) -> "BaseTestServer":
await self.start_server(loop=self._loop)
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
await self.close()
class TestServer(BaseTestServer):
def __init__(
self,
app: Application,
*,
scheme: str = "",
host: str = "127.0.0.1",
port: int | None = None,
**kwargs: Any,
):
self.app = app
super().__init__(scheme=scheme, host=host, port=port, **kwargs)
async def _make_runner(self, **kwargs: Any) -> BaseRunner:
return AppRunner(self.app, **kwargs)
class RawTestServer(BaseTestServer):
def __init__(
self,
handler: _RequestHandler,
*,
scheme: str = "",
host: str = "127.0.0.1",
port: int | None = None,
**kwargs: Any,
) -> None:
self._handler = handler
super().__init__(scheme=scheme, host=host, port=port, **kwargs)
async def _make_runner(self, debug: bool = True, **kwargs: Any) -> ServerRunner:
srv = Server(self._handler, loop=self._loop, debug=debug, **kwargs)
return ServerRunner(srv, debug=debug, **kwargs)
class TestClient(Generic[_Request, _ApplicationNone]):
"""
A test client implementation.
To write functional tests for aiohttp based servers.
"""
__test__ = False
@overload
def __init__(
self: "TestClient[Request, Application]",
server: TestServer,
*,
cookie_jar: AbstractCookieJar | None = None,
**kwargs: Any,
) -> None: ...
@overload
def __init__(
self: "TestClient[_Request, None]",
server: BaseTestServer,
*,
cookie_jar: AbstractCookieJar | None = None,
**kwargs: Any,
) -> None: ...
def __init__(
self,
server: BaseTestServer,
*,
cookie_jar: AbstractCookieJar | None = None,
loop: asyncio.AbstractEventLoop | None = None,
**kwargs: Any,
) -> None:
if not isinstance(server, BaseTestServer):
raise TypeError(
"server must be TestServer instance, found type: %r" % type(server)
)
self._server = server
self._loop = loop
if cookie_jar is None:
cookie_jar = aiohttp.CookieJar(unsafe=True, loop=loop)
self._session = ClientSession(loop=loop, cookie_jar=cookie_jar, **kwargs)
self._session._retry_connection = False
self._closed = False
self._responses: list[ClientResponse] = []
self._websockets: list[ClientWebSocketResponse[bool]] = []
async def start_server(self) -> None:
await self._server.start_server(loop=self._loop)
@property
def host(self) -> str:
return self._server.host
@property
def port(self) -> int | None:
return self._server.port
@property
def server(self) -> BaseTestServer:
return self._server
@property
def app(self) -> _ApplicationNone:
return getattr(self._server, "app", None) # type: ignore[return-value]
@property
def session(self) -> ClientSession:
"""An internal aiohttp.ClientSession.
Unlike the methods on the TestClient, client session requests
do not automatically include the host in the url queried, and
will require an absolute path to the resource.
"""
return self._session
def make_url(self, path: StrOrURL) -> URL:
return self._server.make_url(path)
async def _request(
self, method: str, path: StrOrURL, **kwargs: Any
) -> ClientResponse:
resp = await self._session.request(method, self.make_url(path), **kwargs)
# save it to close later
self._responses.append(resp)
return resp
if sys.version_info >= (3, 11) and TYPE_CHECKING:
def request(
self, method: str, path: StrOrURL, **kwargs: Unpack[_RequestOptions]
) -> _RequestContextManager: ...
def get(
self,
path: StrOrURL,
**kwargs: Unpack[_RequestOptions],
) -> _RequestContextManager: ...
def options(
self,
path: StrOrURL,
**kwargs: Unpack[_RequestOptions],
) -> _RequestContextManager: ...
def head(
self,
path: StrOrURL,
**kwargs: Unpack[_RequestOptions],
) -> _RequestContextManager: ...
def post(
self,
path: StrOrURL,
**kwargs: Unpack[_RequestOptions],
) -> _RequestContextManager: ...
def put(
self,
path: StrOrURL,
**kwargs: Unpack[_RequestOptions],
) -> _RequestContextManager: ...
def patch(
self,
path: StrOrURL,
**kwargs: Unpack[_RequestOptions],
) -> _RequestContextManager: ...
def delete(
self,
path: StrOrURL,
**kwargs: Unpack[_RequestOptions],
) -> _RequestContextManager: ...
else:
def request(
self, method: str, path: StrOrURL, **kwargs: Any
) -> _RequestContextManager:
"""Routes a request to tested http server.
The interface is identical to aiohttp.ClientSession.request,
except the loop kwarg is overridden by the instance used by the
test server.
"""
return _RequestContextManager(self._request(method, path, **kwargs))
def get(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager:
"""Perform an HTTP GET request."""
return _RequestContextManager(self._request(hdrs.METH_GET, path, **kwargs))
def post(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager:
"""Perform an HTTP POST request."""
return _RequestContextManager(self._request(hdrs.METH_POST, path, **kwargs))
def options(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager:
"""Perform an HTTP OPTIONS request."""
return _RequestContextManager(
self._request(hdrs.METH_OPTIONS, path, **kwargs)
)
def head(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager:
"""Perform an HTTP HEAD request."""
return _RequestContextManager(self._request(hdrs.METH_HEAD, path, **kwargs))
def put(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager:
"""Perform an HTTP PUT request."""
return _RequestContextManager(self._request(hdrs.METH_PUT, path, **kwargs))
def patch(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager:
"""Perform an HTTP PATCH request."""
return _RequestContextManager(
self._request(hdrs.METH_PATCH, path, **kwargs)
)
def delete(self, path: StrOrURL, **kwargs: Any) -> _RequestContextManager:
"""Perform an HTTP PATCH request."""
return _RequestContextManager(
self._request(hdrs.METH_DELETE, path, **kwargs)
)
@overload
def ws_connect(
self, path: StrOrURL, *, decode_text: Literal[True] = ..., **kwargs: Any
) -> "_BaseRequestContextManager[ClientWebSocketResponse[Literal[True]]]": ...
@overload
def ws_connect(
self, path: StrOrURL, *, decode_text: Literal[False], **kwargs: Any
) -> "_BaseRequestContextManager[ClientWebSocketResponse[Literal[False]]]": ...
@overload
def ws_connect(
self, path: StrOrURL, *, decode_text: bool = ..., **kwargs: Any
) -> "_BaseRequestContextManager[ClientWebSocketResponse[bool]]": ...
def ws_connect(
self, path: StrOrURL, *, decode_text: bool = True, **kwargs: Any
) -> "_BaseRequestContextManager[ClientWebSocketResponse[bool]]":
"""Initiate websocket connection.
The api corresponds to aiohttp.ClientSession.ws_connect.
"""
return _WSRequestContextManager(
self._ws_connect(path, decode_text=decode_text, **kwargs)
)
@overload
async def _ws_connect(
self, path: StrOrURL, *, decode_text: Literal[True] = ..., **kwargs: Any
) -> "ClientWebSocketResponse[Literal[True]]": ...
@overload
async def _ws_connect(
self, path: StrOrURL, *, decode_text: Literal[False], **kwargs: Any
) -> "ClientWebSocketResponse[Literal[False]]": ...
@overload
async def _ws_connect(
self, path: StrOrURL, *, decode_text: bool = ..., **kwargs: Any
) -> "ClientWebSocketResponse[bool]": ...
async def _ws_connect(
self, path: StrOrURL, *, decode_text: bool = True, **kwargs: Any
) -> "ClientWebSocketResponse[bool]":
ws = await self._session.ws_connect(
self.make_url(path), decode_text=decode_text, **kwargs
)
self._websockets.append(ws)
return ws
async def close(self) -> None:
"""Close all fixtures created by the test client.
After that point, the TestClient is no longer usable.
This is an idempotent function: running close multiple times
will not have any additional effects.
close is also run on exit when used as a(n) (asynchronous)
context manager.
"""
if not self._closed:
for resp in self._responses:
resp.close()
for ws in self._websockets:
await ws.close()
await self._session.close()
await self._server.close()
self._closed = True
def __enter__(self) -> None:
raise TypeError("Use async with instead")
def __exit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> None:
# __exit__ should exist in pair with __enter__ but never executed
pass # pragma: no cover
async def __aenter__(self) -> Self:
await self.start_server()
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> None:
await self.close()
class AioHTTPTestCase(IsolatedAsyncioTestCase):
"""A base class to allow for unittest web applications using aiohttp.
Provides the following:
* self.client (aiohttp.test_utils.TestClient): an aiohttp test client.
* self.loop (asyncio.BaseEventLoop): the event loop in which the
application and server are running.
* self.app (aiohttp.web.Application): the application returned by
self.get_application()
Note that the TestClient's methods are asynchronous: you have to
execute function on the test client using asynchronous methods.
"""
async def get_application(self) -> Application:
"""Get application.
This method should be overridden
to return the aiohttp.web.Application
object to test.
"""
return self.get_app()
def get_app(self) -> Application:
"""Obsolete method used to constructing web application.
Use .get_application() coroutine instead.
"""
raise RuntimeError("Did you forget to define get_application()?")
async def asyncSetUp(self) -> None:
self.loop = asyncio.get_running_loop()
return await self.setUpAsync()
async def setUpAsync(self) -> None:
self.app = await self.get_application()
self.server = await self.get_server(self.app)
self.client = await self.get_client(self.server)
await self.client.start_server()
async def asyncTearDown(self) -> None:
return await self.tearDownAsync()
async def tearDownAsync(self) -> None:
await self.client.close()
async def get_server(self, app: Application) -> TestServer:
"""Return a TestServer instance."""
return TestServer(app, loop=self.loop)
async def get_client(self, server: TestServer) -> TestClient[Request, Application]:
"""Return a TestClient instance."""
return TestClient(server, loop=self.loop)
def unittest_run_loop(func: Any, *args: Any, **kwargs: Any) -> Any:
"""
A decorator dedicated to use with asynchronous AioHTTPTestCase test methods.
In 3.8+, this does nothing.
"""
warnings.warn(
"Decorator `@unittest_run_loop` is no longer needed in aiohttp 3.8+",
DeprecationWarning,
stacklevel=2,
)
return func
_LOOP_FACTORY = Callable[[], asyncio.AbstractEventLoop]
@contextlib.contextmanager
def loop_context(
loop_factory: _LOOP_FACTORY = asyncio.new_event_loop, fast: bool = False
) -> Iterator[asyncio.AbstractEventLoop]:
"""A contextmanager that creates an event_loop, for test purposes.
Handles the creation and cleanup of a test loop.
"""
loop = setup_test_loop(loop_factory)
yield loop
teardown_test_loop(loop, fast=fast)
def setup_test_loop(
loop_factory: _LOOP_FACTORY = asyncio.new_event_loop,
) -> asyncio.AbstractEventLoop:
"""Create and return an asyncio.BaseEventLoop instance.
The caller should also call teardown_test_loop,
once they are done with the loop.
"""
loop = loop_factory()
asyncio.set_event_loop(loop)
return loop
def teardown_test_loop(loop: asyncio.AbstractEventLoop, fast: bool = False) -> None:
"""Teardown and cleanup an event_loop created by setup_test_loop."""
closed = loop.is_closed()
if not closed:
loop.call_soon(loop.stop)
loop.run_forever()
loop.close()
if not fast:
gc.collect()
asyncio.set_event_loop(None)
def _create_app_mock() -> mock.MagicMock:
def get_dict(app: Any, key: str) -> Any:
return app.__app_dict[key]
def set_dict(app: Any, key: str, value: Any) -> None:
app.__app_dict[key] = value
app = mock.MagicMock(spec=Application)
app.__app_dict = {}
app.__getitem__ = get_dict
app.__setitem__ = set_dict
app._debug = False
app.on_response_prepare = Signal(app)
app.on_response_prepare.freeze()
return app
def _create_transport(sslcontext: SSLContext | None = None) -> mock.Mock:
transport = mock.Mock()
def get_extra_info(key: str) -> SSLContext | tuple[str, int] | None:
if key == "sslcontext":
return sslcontext
return ("127.0.0.1", 80) if key == "sockname" else None
transport.get_extra_info.side_effect = get_extra_info
return transport
def make_mocked_request(
method: str,
path: str,
headers: Any = None,
*,
match_info: Any = sentinel,
version: HttpVersion = HttpVersion(1, 1),
closing: bool = False,
app: Any = None,
writer: Any = sentinel,
protocol: Any = sentinel,
transport: Any = sentinel,
payload: StreamReader = EMPTY_PAYLOAD,
sslcontext: SSLContext | None = None,
client_max_size: int = 1024**2,
loop: Any = ...,
) -> Request:
"""Creates mocked web.Request testing purposes.
Useful in unit tests, when spinning full web server is overkill or
specific conditions and errors are hard to trigger.
"""
task = mock.Mock()
if loop is ...:
# no loop passed, try to get the current one if
# its is running as we need a real loop to create
# executor jobs to be able to do testing
# with a real executor
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = mock.Mock()
loop.create_future.return_value = ()
if version < HttpVersion(1, 1):
closing = True
if headers:
headers = CIMultiDictProxy(CIMultiDict(headers))
raw_hdrs = tuple(
(k.encode("utf-8"), v.encode("utf-8")) for k, v in headers.items()
)
else:
headers = CIMultiDictProxy(CIMultiDict())
raw_hdrs = ()
chunked = "chunked" in headers.get(hdrs.TRANSFER_ENCODING, "").lower()
upgrade = headers.get(hdrs.CONNECTION, "").lower() == "upgrade" and bool(
headers.get(hdrs.UPGRADE)
)
message = RawRequestMessage(
method.upper(),
path,
version,
headers,
raw_hdrs,
closing,
None,
upgrade,
chunked,
URL(path),
)
if app is None:
app = _create_app_mock()
if transport is sentinel:
transport = _create_transport(sslcontext)
if protocol is sentinel:
protocol = mock.Mock()
protocol.max_field_size = 8190
protocol.max_line_length = 8190
protocol.max_headers = 128
protocol.transport = transport
type(protocol).peername = mock.PropertyMock(
return_value=transport.get_extra_info("peername")
)
type(protocol).sockname = mock.PropertyMock(
return_value=transport.get_extra_info("sockname")
)
type(protocol).ssl_context = mock.PropertyMock(return_value=sslcontext)
if writer is sentinel:
writer = mock.Mock()
writer.write_headers = make_mocked_coro(None)
writer.write = make_mocked_coro(None)
writer.write_eof = make_mocked_coro(None)
writer.drain = make_mocked_coro(None)
writer.transport = transport
protocol.transport = transport
protocol.writer = writer
req = Request(
message, payload, protocol, writer, task, loop, client_max_size=client_max_size
)
match_info = UrlMappingMatchInfo(
{} if match_info is sentinel else match_info, mock.Mock()
)
match_info.add_app(app)
req._match_info = match_info
return req
def make_mocked_coro(
return_value: Any = sentinel, raise_exception: Any = sentinel
) -> Any:
"""Creates a coroutine mock."""
async def mock_coro(*args: Any, **kwargs: Any) -> Any:
if raise_exception is not sentinel:
raise raise_exception
if not inspect.isawaitable(return_value):
return return_value
await return_value
return mock.Mock(wraps=mock_coro)
+453
View File
@@ -0,0 +1,453 @@
from types import SimpleNamespace
from typing import TYPE_CHECKING, Any, TypeVar
import attr
from aiosignal import Signal
from multidict import CIMultiDict
from yarl import URL
from .client_reqrep import ClientResponse
if TYPE_CHECKING:
from .client import ClientSession
_ParamT_contra = TypeVar("_ParamT_contra", contravariant=True)
_TracingSignal = Signal[ClientSession, SimpleNamespace, _ParamT_contra]
__all__ = (
"TraceConfig",
"TraceRequestStartParams",
"TraceRequestEndParams",
"TraceRequestExceptionParams",
"TraceConnectionQueuedStartParams",
"TraceConnectionQueuedEndParams",
"TraceConnectionCreateStartParams",
"TraceConnectionCreateEndParams",
"TraceConnectionReuseconnParams",
"TraceDnsResolveHostStartParams",
"TraceDnsResolveHostEndParams",
"TraceDnsCacheHitParams",
"TraceDnsCacheMissParams",
"TraceRequestRedirectParams",
"TraceRequestChunkSentParams",
"TraceResponseChunkReceivedParams",
"TraceRequestHeadersSentParams",
)
class TraceConfig:
"""First-class used to trace requests launched via ClientSession objects."""
def __init__(
self, trace_config_ctx_factory: type[SimpleNamespace] = SimpleNamespace
) -> None:
self._on_request_start: _TracingSignal[TraceRequestStartParams] = Signal(self)
self._on_request_chunk_sent: _TracingSignal[TraceRequestChunkSentParams] = (
Signal(self)
)
self._on_response_chunk_received: _TracingSignal[
TraceResponseChunkReceivedParams
] = Signal(self)
self._on_request_end: _TracingSignal[TraceRequestEndParams] = Signal(self)
self._on_request_exception: _TracingSignal[TraceRequestExceptionParams] = (
Signal(self)
)
self._on_request_redirect: _TracingSignal[TraceRequestRedirectParams] = Signal(
self
)
self._on_connection_queued_start: _TracingSignal[
TraceConnectionQueuedStartParams
] = Signal(self)
self._on_connection_queued_end: _TracingSignal[
TraceConnectionQueuedEndParams
] = Signal(self)
self._on_connection_create_start: _TracingSignal[
TraceConnectionCreateStartParams
] = Signal(self)
self._on_connection_create_end: _TracingSignal[
TraceConnectionCreateEndParams
] = Signal(self)
self._on_connection_reuseconn: _TracingSignal[
TraceConnectionReuseconnParams
] = Signal(self)
self._on_dns_resolvehost_start: _TracingSignal[
TraceDnsResolveHostStartParams
] = Signal(self)
self._on_dns_resolvehost_end: _TracingSignal[TraceDnsResolveHostEndParams] = (
Signal(self)
)
self._on_dns_cache_hit: _TracingSignal[TraceDnsCacheHitParams] = Signal(self)
self._on_dns_cache_miss: _TracingSignal[TraceDnsCacheMissParams] = Signal(self)
self._on_request_headers_sent: _TracingSignal[TraceRequestHeadersSentParams] = (
Signal(self)
)
self._trace_config_ctx_factory = trace_config_ctx_factory
def trace_config_ctx(self, trace_request_ctx: Any = None) -> SimpleNamespace:
"""Return a new trace_config_ctx instance"""
return self._trace_config_ctx_factory(trace_request_ctx=trace_request_ctx)
def freeze(self) -> None:
self._on_request_start.freeze()
self._on_request_chunk_sent.freeze()
self._on_response_chunk_received.freeze()
self._on_request_end.freeze()
self._on_request_exception.freeze()
self._on_request_redirect.freeze()
self._on_connection_queued_start.freeze()
self._on_connection_queued_end.freeze()
self._on_connection_create_start.freeze()
self._on_connection_create_end.freeze()
self._on_connection_reuseconn.freeze()
self._on_dns_resolvehost_start.freeze()
self._on_dns_resolvehost_end.freeze()
self._on_dns_cache_hit.freeze()
self._on_dns_cache_miss.freeze()
self._on_request_headers_sent.freeze()
@property
def on_request_start(self) -> "_TracingSignal[TraceRequestStartParams]":
return self._on_request_start
@property
def on_request_chunk_sent(
self,
) -> "_TracingSignal[TraceRequestChunkSentParams]":
return self._on_request_chunk_sent
@property
def on_response_chunk_received(
self,
) -> "_TracingSignal[TraceResponseChunkReceivedParams]":
return self._on_response_chunk_received
@property
def on_request_end(self) -> "_TracingSignal[TraceRequestEndParams]":
return self._on_request_end
@property
def on_request_exception(
self,
) -> "_TracingSignal[TraceRequestExceptionParams]":
return self._on_request_exception
@property
def on_request_redirect(
self,
) -> "_TracingSignal[TraceRequestRedirectParams]":
return self._on_request_redirect
@property
def on_connection_queued_start(
self,
) -> "_TracingSignal[TraceConnectionQueuedStartParams]":
return self._on_connection_queued_start
@property
def on_connection_queued_end(
self,
) -> "_TracingSignal[TraceConnectionQueuedEndParams]":
return self._on_connection_queued_end
@property
def on_connection_create_start(
self,
) -> "_TracingSignal[TraceConnectionCreateStartParams]":
return self._on_connection_create_start
@property
def on_connection_create_end(
self,
) -> "_TracingSignal[TraceConnectionCreateEndParams]":
return self._on_connection_create_end
@property
def on_connection_reuseconn(
self,
) -> "_TracingSignal[TraceConnectionReuseconnParams]":
return self._on_connection_reuseconn
@property
def on_dns_resolvehost_start(
self,
) -> "_TracingSignal[TraceDnsResolveHostStartParams]":
return self._on_dns_resolvehost_start
@property
def on_dns_resolvehost_end(
self,
) -> "_TracingSignal[TraceDnsResolveHostEndParams]":
return self._on_dns_resolvehost_end
@property
def on_dns_cache_hit(self) -> "_TracingSignal[TraceDnsCacheHitParams]":
return self._on_dns_cache_hit
@property
def on_dns_cache_miss(self) -> "_TracingSignal[TraceDnsCacheMissParams]":
return self._on_dns_cache_miss
@property
def on_request_headers_sent(
self,
) -> "_TracingSignal[TraceRequestHeadersSentParams]":
return self._on_request_headers_sent
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceRequestStartParams:
"""Parameters sent by the `on_request_start` signal"""
method: str
url: URL
headers: "CIMultiDict[str]"
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceRequestChunkSentParams:
"""Parameters sent by the `on_request_chunk_sent` signal"""
method: str
url: URL
chunk: bytes
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceResponseChunkReceivedParams:
"""Parameters sent by the `on_response_chunk_received` signal"""
method: str
url: URL
chunk: bytes
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceRequestEndParams:
"""Parameters sent by the `on_request_end` signal"""
method: str
url: URL
headers: "CIMultiDict[str]"
response: ClientResponse
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceRequestExceptionParams:
"""Parameters sent by the `on_request_exception` signal"""
method: str
url: URL
headers: "CIMultiDict[str]"
exception: BaseException
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceRequestRedirectParams:
"""Parameters sent by the `on_request_redirect` signal"""
method: str
url: URL
headers: "CIMultiDict[str]"
response: ClientResponse
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceConnectionQueuedStartParams:
"""Parameters sent by the `on_connection_queued_start` signal"""
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceConnectionQueuedEndParams:
"""Parameters sent by the `on_connection_queued_end` signal"""
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceConnectionCreateStartParams:
"""Parameters sent by the `on_connection_create_start` signal"""
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceConnectionCreateEndParams:
"""Parameters sent by the `on_connection_create_end` signal"""
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceConnectionReuseconnParams:
"""Parameters sent by the `on_connection_reuseconn` signal"""
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceDnsResolveHostStartParams:
"""Parameters sent by the `on_dns_resolvehost_start` signal"""
host: str
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceDnsResolveHostEndParams:
"""Parameters sent by the `on_dns_resolvehost_end` signal"""
host: str
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceDnsCacheHitParams:
"""Parameters sent by the `on_dns_cache_hit` signal"""
host: str
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceDnsCacheMissParams:
"""Parameters sent by the `on_dns_cache_miss` signal"""
host: str
@attr.s(auto_attribs=True, frozen=True, slots=True)
class TraceRequestHeadersSentParams:
"""Parameters sent by the `on_request_headers_sent` signal"""
method: str
url: URL
headers: "CIMultiDict[str]"
class Trace:
"""Internal dependency holder class.
Used to keep together the main dependencies used
at the moment of send a signal.
"""
def __init__(
self,
session: "ClientSession",
trace_config: TraceConfig,
trace_config_ctx: SimpleNamespace,
) -> None:
self._trace_config = trace_config
self._trace_config_ctx = trace_config_ctx
self._session = session
async def send_request_start(
self, method: str, url: URL, headers: "CIMultiDict[str]"
) -> None:
return await self._trace_config.on_request_start.send(
self._session,
self._trace_config_ctx,
TraceRequestStartParams(method, url, headers),
)
async def send_request_chunk_sent(
self, method: str, url: URL, chunk: bytes
) -> None:
return await self._trace_config.on_request_chunk_sent.send(
self._session,
self._trace_config_ctx,
TraceRequestChunkSentParams(method, url, chunk),
)
async def send_response_chunk_received(
self, method: str, url: URL, chunk: bytes
) -> None:
return await self._trace_config.on_response_chunk_received.send(
self._session,
self._trace_config_ctx,
TraceResponseChunkReceivedParams(method, url, chunk),
)
async def send_request_end(
self,
method: str,
url: URL,
headers: "CIMultiDict[str]",
response: ClientResponse,
) -> None:
return await self._trace_config.on_request_end.send(
self._session,
self._trace_config_ctx,
TraceRequestEndParams(method, url, headers, response),
)
async def send_request_exception(
self,
method: str,
url: URL,
headers: "CIMultiDict[str]",
exception: BaseException,
) -> None:
return await self._trace_config.on_request_exception.send(
self._session,
self._trace_config_ctx,
TraceRequestExceptionParams(method, url, headers, exception),
)
async def send_request_redirect(
self,
method: str,
url: URL,
headers: "CIMultiDict[str]",
response: ClientResponse,
) -> None:
return await self._trace_config._on_request_redirect.send(
self._session,
self._trace_config_ctx,
TraceRequestRedirectParams(method, url, headers, response),
)
async def send_connection_queued_start(self) -> None:
return await self._trace_config.on_connection_queued_start.send(
self._session, self._trace_config_ctx, TraceConnectionQueuedStartParams()
)
async def send_connection_queued_end(self) -> None:
return await self._trace_config.on_connection_queued_end.send(
self._session, self._trace_config_ctx, TraceConnectionQueuedEndParams()
)
async def send_connection_create_start(self) -> None:
return await self._trace_config.on_connection_create_start.send(
self._session, self._trace_config_ctx, TraceConnectionCreateStartParams()
)
async def send_connection_create_end(self) -> None:
return await self._trace_config.on_connection_create_end.send(
self._session, self._trace_config_ctx, TraceConnectionCreateEndParams()
)
async def send_connection_reuseconn(self) -> None:
return await self._trace_config.on_connection_reuseconn.send(
self._session, self._trace_config_ctx, TraceConnectionReuseconnParams()
)
async def send_dns_resolvehost_start(self, host: str) -> None:
return await self._trace_config.on_dns_resolvehost_start.send(
self._session, self._trace_config_ctx, TraceDnsResolveHostStartParams(host)
)
async def send_dns_resolvehost_end(self, host: str) -> None:
return await self._trace_config.on_dns_resolvehost_end.send(
self._session, self._trace_config_ctx, TraceDnsResolveHostEndParams(host)
)
async def send_dns_cache_hit(self, host: str) -> None:
return await self._trace_config.on_dns_cache_hit.send(
self._session, self._trace_config_ctx, TraceDnsCacheHitParams(host)
)
async def send_dns_cache_miss(self, host: str) -> None:
return await self._trace_config.on_dns_cache_miss.send(
self._session, self._trace_config_ctx, TraceDnsCacheMissParams(host)
)
async def send_request_headers(
self, method: str, url: URL, headers: "CIMultiDict[str]"
) -> None:
return await self._trace_config._on_request_headers_sent.send(
self._session,
self._trace_config_ctx,
TraceRequestHeadersSentParams(method, url, headers),
)
+61
View File
@@ -0,0 +1,61 @@
import json
import os
from collections.abc import Awaitable, Callable, Iterable, Mapping
from typing import TYPE_CHECKING, Any, Protocol, Union
from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy, istr
from yarl import URL, Query as _Query
Query = _Query
DEFAULT_JSON_ENCODER = json.dumps
DEFAULT_JSON_DECODER = json.loads
if TYPE_CHECKING:
_CIMultiDict = CIMultiDict[str]
_CIMultiDictProxy = CIMultiDictProxy[str]
_MultiDict = MultiDict[str]
_MultiDictProxy = MultiDictProxy[str]
from http.cookies import BaseCookie, Morsel
from .web import Request, StreamResponse
else:
_CIMultiDict = CIMultiDict
_CIMultiDictProxy = CIMultiDictProxy
_MultiDict = MultiDict
_MultiDictProxy = MultiDictProxy
Byteish = Union[bytes, bytearray, memoryview]
JSONEncoder = Callable[[Any], str]
JSONBytesEncoder = Callable[[Any], bytes]
JSONDecoder = Callable[[str], Any]
LooseHeaders = Union[
Mapping[str, str],
Mapping[istr, str],
_CIMultiDict,
_CIMultiDictProxy,
Iterable[tuple[str | istr, str]],
]
RawHeaders = tuple[tuple[bytes, bytes], ...]
StrOrURL = Union[str, URL]
LooseCookiesMappings = Mapping[str, Union[str, "BaseCookie[str]", "Morsel[Any]"]]
LooseCookiesIterables = Iterable[
tuple[str, Union[str, "BaseCookie[str]", "Morsel[Any]"]]
]
LooseCookies = Union[
LooseCookiesMappings,
LooseCookiesIterables,
"BaseCookie[str]",
]
Handler = Callable[["Request"], Awaitable["StreamResponse"]]
class Middleware(Protocol):
def __call__(
self, request: "Request", handler: Handler
) -> Awaitable["StreamResponse"]: ...
PathLike = Union[str, "os.PathLike[str]"]
+593
View File
@@ -0,0 +1,593 @@
import asyncio
import logging
import os
import socket
import sys
import warnings
from argparse import ArgumentParser
from collections.abc import Awaitable, Callable, Iterable, Iterable as TypingIterable
from contextlib import suppress
from importlib import import_module
from typing import TYPE_CHECKING, Any, cast
from .abc import AbstractAccessLogger
from .helpers import AppKey, RequestKey, ResponseKey
from .log import access_logger
from .typedefs import PathLike
from .web_app import Application as Application, CleanupError as CleanupError
from .web_exceptions import (
HTTPAccepted as HTTPAccepted,
HTTPBadGateway as HTTPBadGateway,
HTTPBadRequest as HTTPBadRequest,
HTTPClientError as HTTPClientError,
HTTPConflict as HTTPConflict,
HTTPCreated as HTTPCreated,
HTTPError as HTTPError,
HTTPException as HTTPException,
HTTPExpectationFailed as HTTPExpectationFailed,
HTTPFailedDependency as HTTPFailedDependency,
HTTPForbidden as HTTPForbidden,
HTTPFound as HTTPFound,
HTTPGatewayTimeout as HTTPGatewayTimeout,
HTTPGone as HTTPGone,
HTTPInsufficientStorage as HTTPInsufficientStorage,
HTTPInternalServerError as HTTPInternalServerError,
HTTPLengthRequired as HTTPLengthRequired,
HTTPMethodNotAllowed as HTTPMethodNotAllowed,
HTTPMisdirectedRequest as HTTPMisdirectedRequest,
HTTPMove as HTTPMove,
HTTPMovedPermanently as HTTPMovedPermanently,
HTTPMultipleChoices as HTTPMultipleChoices,
HTTPNetworkAuthenticationRequired as HTTPNetworkAuthenticationRequired,
HTTPNoContent as HTTPNoContent,
HTTPNonAuthoritativeInformation as HTTPNonAuthoritativeInformation,
HTTPNotAcceptable as HTTPNotAcceptable,
HTTPNotExtended as HTTPNotExtended,
HTTPNotFound as HTTPNotFound,
HTTPNotImplemented as HTTPNotImplemented,
HTTPNotModified as HTTPNotModified,
HTTPOk as HTTPOk,
HTTPPartialContent as HTTPPartialContent,
HTTPPaymentRequired as HTTPPaymentRequired,
HTTPPermanentRedirect as HTTPPermanentRedirect,
HTTPPreconditionFailed as HTTPPreconditionFailed,
HTTPPreconditionRequired as HTTPPreconditionRequired,
HTTPProxyAuthenticationRequired as HTTPProxyAuthenticationRequired,
HTTPRedirection as HTTPRedirection,
HTTPRequestEntityTooLarge as HTTPRequestEntityTooLarge,
HTTPRequestHeaderFieldsTooLarge as HTTPRequestHeaderFieldsTooLarge,
HTTPRequestRangeNotSatisfiable as HTTPRequestRangeNotSatisfiable,
HTTPRequestTimeout as HTTPRequestTimeout,
HTTPRequestURITooLong as HTTPRequestURITooLong,
HTTPResetContent as HTTPResetContent,
HTTPSeeOther as HTTPSeeOther,
HTTPServerError as HTTPServerError,
HTTPServiceUnavailable as HTTPServiceUnavailable,
HTTPSuccessful as HTTPSuccessful,
HTTPTemporaryRedirect as HTTPTemporaryRedirect,
HTTPTooManyRequests as HTTPTooManyRequests,
HTTPUnauthorized as HTTPUnauthorized,
HTTPUnavailableForLegalReasons as HTTPUnavailableForLegalReasons,
HTTPUnprocessableEntity as HTTPUnprocessableEntity,
HTTPUnsupportedMediaType as HTTPUnsupportedMediaType,
HTTPUpgradeRequired as HTTPUpgradeRequired,
HTTPUseProxy as HTTPUseProxy,
HTTPVariantAlsoNegotiates as HTTPVariantAlsoNegotiates,
HTTPVersionNotSupported as HTTPVersionNotSupported,
NotAppKeyWarning as NotAppKeyWarning,
)
from .web_fileresponse import FileResponse as FileResponse
from .web_log import AccessLogger
from .web_middlewares import (
middleware as middleware,
normalize_path_middleware as normalize_path_middleware,
)
from .web_protocol import (
PayloadAccessError as PayloadAccessError,
RequestHandler as RequestHandler,
RequestPayloadError as RequestPayloadError,
)
from .web_request import (
BaseRequest as BaseRequest,
FileField as FileField,
Request as Request,
)
from .web_response import (
ContentCoding as ContentCoding,
Response as Response,
StreamResponse as StreamResponse,
json_bytes_response as json_bytes_response,
json_response as json_response,
)
from .web_routedef import (
AbstractRouteDef as AbstractRouteDef,
RouteDef as RouteDef,
RouteTableDef as RouteTableDef,
StaticDef as StaticDef,
delete as delete,
get as get,
head as head,
options as options,
patch as patch,
post as post,
put as put,
route as route,
static as static,
view as view,
)
from .web_runner import (
AppRunner as AppRunner,
BaseRunner as BaseRunner,
BaseSite as BaseSite,
GracefulExit as GracefulExit,
NamedPipeSite as NamedPipeSite,
ServerRunner as ServerRunner,
SockSite as SockSite,
TCPSite as TCPSite,
UnixSite as UnixSite,
)
from .web_server import Server as Server
from .web_urldispatcher import (
AbstractResource as AbstractResource,
AbstractRoute as AbstractRoute,
DynamicResource as DynamicResource,
PlainResource as PlainResource,
PrefixedSubAppResource as PrefixedSubAppResource,
Resource as Resource,
ResourceRoute as ResourceRoute,
StaticResource as StaticResource,
UrlDispatcher as UrlDispatcher,
UrlMappingMatchInfo as UrlMappingMatchInfo,
View as View,
)
from .web_ws import (
WebSocketReady as WebSocketReady,
WebSocketResponse as WebSocketResponse,
WSMsgType as WSMsgType,
)
__all__ = (
# web_app
"AppKey",
"Application",
"CleanupError",
# web_exceptions
"NotAppKeyWarning",
"HTTPAccepted",
"HTTPBadGateway",
"HTTPBadRequest",
"HTTPClientError",
"HTTPConflict",
"HTTPCreated",
"HTTPError",
"HTTPException",
"HTTPExpectationFailed",
"HTTPFailedDependency",
"HTTPForbidden",
"HTTPFound",
"HTTPGatewayTimeout",
"HTTPGone",
"HTTPInsufficientStorage",
"HTTPInternalServerError",
"HTTPLengthRequired",
"HTTPMethodNotAllowed",
"HTTPMisdirectedRequest",
"HTTPMove",
"HTTPMovedPermanently",
"HTTPMultipleChoices",
"HTTPNetworkAuthenticationRequired",
"HTTPNoContent",
"HTTPNonAuthoritativeInformation",
"HTTPNotAcceptable",
"HTTPNotExtended",
"HTTPNotFound",
"HTTPNotImplemented",
"HTTPNotModified",
"HTTPOk",
"HTTPPartialContent",
"HTTPPaymentRequired",
"HTTPPermanentRedirect",
"HTTPPreconditionFailed",
"HTTPPreconditionRequired",
"HTTPProxyAuthenticationRequired",
"HTTPRedirection",
"HTTPRequestEntityTooLarge",
"HTTPRequestHeaderFieldsTooLarge",
"HTTPRequestRangeNotSatisfiable",
"HTTPRequestTimeout",
"HTTPRequestURITooLong",
"HTTPResetContent",
"HTTPSeeOther",
"HTTPServerError",
"HTTPServiceUnavailable",
"HTTPSuccessful",
"HTTPTemporaryRedirect",
"HTTPTooManyRequests",
"HTTPUnauthorized",
"HTTPUnavailableForLegalReasons",
"HTTPUnprocessableEntity",
"HTTPUnsupportedMediaType",
"HTTPUpgradeRequired",
"HTTPUseProxy",
"HTTPVariantAlsoNegotiates",
"HTTPVersionNotSupported",
# web_fileresponse
"FileResponse",
# web_middlewares
"middleware",
"normalize_path_middleware",
# web_protocol
"PayloadAccessError",
"RequestHandler",
"RequestPayloadError",
# web_request
"BaseRequest",
"FileField",
"Request",
"RequestKey",
# web_response
"ContentCoding",
"Response",
"StreamResponse",
"json_bytes_response",
"json_response",
"ResponseKey",
# web_routedef
"AbstractRouteDef",
"RouteDef",
"RouteTableDef",
"StaticDef",
"delete",
"get",
"head",
"options",
"patch",
"post",
"put",
"route",
"static",
"view",
# web_runner
"AppRunner",
"BaseRunner",
"BaseSite",
"GracefulExit",
"ServerRunner",
"SockSite",
"TCPSite",
"UnixSite",
"NamedPipeSite",
# web_server
"Server",
# web_urldispatcher
"AbstractResource",
"AbstractRoute",
"DynamicResource",
"PlainResource",
"PrefixedSubAppResource",
"Resource",
"ResourceRoute",
"StaticResource",
"UrlDispatcher",
"UrlMappingMatchInfo",
"View",
# web_ws
"WebSocketReady",
"WebSocketResponse",
"WSMsgType",
# web
"run_app",
)
if TYPE_CHECKING:
from ssl import SSLContext
else:
try:
from ssl import SSLContext
except ImportError: # pragma: no cover
SSLContext = object # type: ignore[misc,assignment]
# Only display warning when using -Wdefault, -We, -X dev or similar.
warnings.filterwarnings("ignore", category=NotAppKeyWarning, append=True)
HostSequence = TypingIterable[str]
async def _run_app(
app: Application | Awaitable[Application],
*,
host: str | HostSequence | None = None,
port: int | None = None,
path: PathLike | TypingIterable[PathLike] | None = None,
sock: socket.socket | TypingIterable[socket.socket] | None = None,
ssl_context: SSLContext | None = None,
print: Callable[..., None] | None = print,
backlog: int = 128,
reuse_address: bool | None = None,
reuse_port: bool | None = None,
**kwargs: Any, # TODO(PY311): Use Unpack
) -> None:
# An internal function to actually do all dirty job for application running
if asyncio.iscoroutine(app):
app = await app
app = cast(Application, app)
runner = AppRunner(app, **kwargs)
await runner.setup()
sites: list[BaseSite] = []
try:
if host is not None:
if isinstance(host, str):
sites.append(
TCPSite(
runner,
host,
port,
ssl_context=ssl_context,
backlog=backlog,
reuse_address=reuse_address,
reuse_port=reuse_port,
)
)
else:
for h in host:
sites.append(
TCPSite(
runner,
h,
port,
ssl_context=ssl_context,
backlog=backlog,
reuse_address=reuse_address,
reuse_port=reuse_port,
)
)
elif path is None and sock is None or port is not None:
sites.append(
TCPSite(
runner,
port=port,
ssl_context=ssl_context,
backlog=backlog,
reuse_address=reuse_address,
reuse_port=reuse_port,
)
)
if path is not None:
if isinstance(path, (str, os.PathLike)):
sites.append(
UnixSite(
runner,
path,
ssl_context=ssl_context,
backlog=backlog,
)
)
else:
for p in path:
sites.append(
UnixSite(
runner,
p,
ssl_context=ssl_context,
backlog=backlog,
)
)
if sock is not None:
if not isinstance(sock, Iterable):
sites.append(
SockSite(
runner,
sock,
ssl_context=ssl_context,
backlog=backlog,
)
)
else:
for s in sock:
sites.append(
SockSite(
runner,
s,
ssl_context=ssl_context,
backlog=backlog,
)
)
for site in sites:
await site.start()
if print: # pragma: no branch
names = sorted(str(s.name) for s in runner.sites)
print(
"======== Running on {} ========\n"
"(Press CTRL+C to quit)".format(", ".join(names))
)
# sleep forever by 1 hour intervals,
while True:
await asyncio.sleep(3600)
finally:
await runner.cleanup()
def _cancel_tasks(
to_cancel: set["asyncio.Task[Any]"], loop: asyncio.AbstractEventLoop
) -> None:
if not to_cancel:
return
for task in to_cancel:
task.cancel()
loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True))
for task in to_cancel:
if task.cancelled():
continue
if task.exception() is not None:
loop.call_exception_handler(
{
"message": "unhandled exception during asyncio.run() shutdown",
"exception": task.exception(),
"task": task,
}
)
def run_app(
app: Application | Awaitable[Application],
*,
host: str | HostSequence | None = None,
port: int | None = None,
path: PathLike | TypingIterable[PathLike] | None = None,
sock: socket.socket | TypingIterable[socket.socket] | None = None,
shutdown_timeout: float = 60.0,
keepalive_timeout: float = 75.0,
ssl_context: SSLContext | None = None,
print: Callable[..., None] | None = print,
backlog: int = 128,
access_log_class: type[AbstractAccessLogger] = AccessLogger,
access_log_format: str = AccessLogger.LOG_FORMAT,
access_log: logging.Logger | None = access_logger,
handle_signals: bool = True,
reuse_address: bool | None = None,
reuse_port: bool | None = None,
handler_cancellation: bool = False,
loop: asyncio.AbstractEventLoop | None = None,
**kwargs: Any,
) -> None:
"""Run an app locally"""
if loop is None:
loop = asyncio.new_event_loop()
# Configure if and only if in debugging mode and using the default logger
if loop.get_debug() and access_log and access_log.name == "aiohttp.access":
if access_log.level == logging.NOTSET:
access_log.setLevel(logging.DEBUG)
if not access_log.hasHandlers():
access_log.addHandler(logging.StreamHandler())
main_task = loop.create_task(
_run_app(
app,
host=host,
port=port,
path=path,
sock=sock,
shutdown_timeout=shutdown_timeout,
keepalive_timeout=keepalive_timeout,
ssl_context=ssl_context,
print=print,
backlog=backlog,
access_log_class=access_log_class,
access_log_format=access_log_format,
access_log=access_log,
handle_signals=handle_signals,
reuse_address=reuse_address,
reuse_port=reuse_port,
handler_cancellation=handler_cancellation,
**kwargs,
)
)
try:
asyncio.set_event_loop(loop)
loop.run_until_complete(main_task)
except (GracefulExit, KeyboardInterrupt): # pragma: no cover
pass
finally:
try:
# Skip when ``main_task`` is already done (e.g. raised during startup).
# Re-running ``loop.run_until_complete`` on a finished task calls
# ``Future.result`` again, which does
# ``raise self._exception.with_traceback(self._exception_tb)`` and
# resets ``exc.__traceback__`` to the originally saved tb — by then
# shallow — clobbering the deep traceback the caller would otherwise
# see (frames from ``cleanup_ctx`` / ``on_startup`` and the user code
# that actually raised).
if not main_task.done():
main_task.cancel()
with suppress(asyncio.CancelledError):
loop.run_until_complete(main_task)
finally:
_cancel_tasks(asyncio.all_tasks(loop), loop)
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
def main(argv: list[str]) -> None:
arg_parser = ArgumentParser(
description="aiohttp.web Application server", prog="aiohttp.web"
)
arg_parser.add_argument(
"entry_func",
help=(
"Callable returning the `aiohttp.web.Application` instance to "
"run. Should be specified in the 'module:function' syntax."
),
metavar="entry-func",
)
arg_parser.add_argument(
"-H",
"--hostname",
help="TCP/IP hostname to serve on (default: localhost)",
default=None,
)
arg_parser.add_argument(
"-P",
"--port",
help="TCP/IP port to serve on (default: %(default)r)",
type=int,
default=8080,
)
arg_parser.add_argument(
"-U",
"--path",
help="Unix file system path to serve on. Can be combined with hostname "
"to serve on both Unix and TCP.",
)
args, extra_argv = arg_parser.parse_known_args(argv)
# Import logic
mod_str, _, func_str = args.entry_func.partition(":")
if not func_str or not mod_str:
arg_parser.error("'entry-func' not in 'module:function' syntax")
if mod_str.startswith("."):
arg_parser.error("relative module names not supported")
try:
module = import_module(mod_str)
except ImportError as ex:
arg_parser.error(f"unable to import {mod_str}: {ex}")
try:
func = getattr(module, func_str)
except AttributeError:
arg_parser.error(f"module {mod_str!r} has no attribute {func_str!r}")
# Compatibility logic
if args.path is not None and not hasattr(socket, "AF_UNIX"):
arg_parser.error(
"file system paths not supported by your operating environment"
)
logging.basicConfig(level=logging.DEBUG)
if args.path and args.hostname is None:
host = port = None
else:
host = args.hostname or "localhost"
port = args.port
app = func(extra_argv)
run_app(app, host=host, port=port, path=args.path)
arg_parser.exit(message="Stopped\n")
if __name__ == "__main__": # pragma: no branch
main(sys.argv[1:]) # pragma: no cover
+610
View File
@@ -0,0 +1,610 @@
import asyncio
import logging
import warnings
from collections.abc import (
AsyncIterator,
Awaitable,
Callable,
Iterable,
Iterator,
Mapping,
MutableMapping,
Sequence,
)
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from functools import lru_cache, partial, update_wrapper
from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast, overload
from aiosignal import Signal
from frozenlist import FrozenList
from . import hdrs
from .abc import (
AbstractAccessLogger,
AbstractMatchInfo,
AbstractRouter,
AbstractStreamWriter,
)
from .helpers import DEBUG, AppKey
from .http_parser import RawRequestMessage
from .log import web_logger
from .streams import StreamReader
from .typedefs import Handler, Middleware
from .web_exceptions import NotAppKeyWarning
from .web_log import AccessLogger
from .web_middlewares import _fix_request_current_app
from .web_protocol import RequestHandler
from .web_request import Request
from .web_response import StreamResponse
from .web_routedef import AbstractRouteDef
from .web_server import Server
from .web_urldispatcher import (
AbstractResource,
AbstractRoute,
Domain,
MaskDomain,
MatchedSubAppResource,
PrefixedSubAppResource,
SystemRoute,
UrlDispatcher,
)
__all__ = ("Application", "CleanupError")
if TYPE_CHECKING:
_AppSignal = Signal["Application"]
_RespPrepareSignal = Signal[Request, StreamResponse]
_Middlewares = FrozenList[Middleware]
_MiddlewaresHandlers = Optional[Sequence[tuple[Middleware, bool]]]
_Subapps = list["Application"]
else:
# No type checker mode, skip types
_AppSignal = Signal
_RespPrepareSignal = Signal
_Middlewares = FrozenList
_MiddlewaresHandlers = Optional[Sequence]
_Subapps = list
_T = TypeVar("_T")
_U = TypeVar("_U")
_Resource = TypeVar("_Resource", bound=AbstractResource)
def _build_middlewares(
handler: Handler, apps: tuple["Application", ...]
) -> Callable[[Request], Awaitable[StreamResponse]]:
"""Apply middlewares to handler."""
for app in apps[::-1]:
for m, _ in app._middlewares_handlers: # type: ignore[union-attr]
handler = update_wrapper(partial(m, handler=handler), handler)
return handler
_cached_build_middleware = lru_cache(maxsize=1024)(_build_middlewares)
class Application(MutableMapping[str | AppKey[Any], Any]):
ATTRS = frozenset(
[
"logger",
"_debug",
"_router",
"_loop",
"_handler_args",
"_middlewares",
"_middlewares_handlers",
"_has_legacy_middlewares",
"_run_middlewares",
"_state",
"_frozen",
"_pre_frozen",
"_subapps",
"_on_response_prepare",
"_on_startup",
"_on_shutdown",
"_on_cleanup",
"_client_max_size",
"_cleanup_ctx",
]
)
def __init__(
self,
*,
logger: logging.Logger = web_logger,
router: UrlDispatcher | None = None,
middlewares: Iterable[Middleware] = (),
handler_args: Mapping[str, Any] | None = None,
client_max_size: int = 1024**2,
loop: asyncio.AbstractEventLoop | None = None,
debug: Any = ..., # mypy doesn't support ellipsis
) -> None:
if router is None:
router = UrlDispatcher()
else:
warnings.warn(
"router argument is deprecated", DeprecationWarning, stacklevel=2
)
assert isinstance(router, AbstractRouter), router
if loop is not None:
warnings.warn(
"loop argument is deprecated", DeprecationWarning, stacklevel=2
)
if debug is not ...:
warnings.warn(
"debug argument is deprecated", DeprecationWarning, stacklevel=2
)
self._debug = debug
self._router: UrlDispatcher = router
self._loop = loop
self._handler_args = handler_args
self.logger = logger
self._middlewares: _Middlewares = FrozenList(middlewares)
# initialized on freezing
self._middlewares_handlers: _MiddlewaresHandlers = None
# initialized on freezing
self._run_middlewares: bool | None = None
self._has_legacy_middlewares: bool = True
self._state: dict[AppKey[Any] | str, object] = {}
self._frozen = False
self._pre_frozen = False
self._subapps: _Subapps = []
self._on_response_prepare: _RespPrepareSignal = Signal(self)
self._on_startup: _AppSignal = Signal(self)
self._on_shutdown: _AppSignal = Signal(self)
self._on_cleanup: _AppSignal = Signal(self)
self._cleanup_ctx = CleanupContext()
self._on_startup.append(self._cleanup_ctx._on_startup)
self._on_cleanup.append(self._cleanup_ctx._on_cleanup)
self._client_max_size = client_max_size
def __init_subclass__(cls: type["Application"]) -> None:
warnings.warn(
f"Inheritance class {cls.__name__} from web.Application is discouraged",
DeprecationWarning,
stacklevel=3,
)
if DEBUG: # pragma: no cover
def __setattr__(self, name: str, val: Any) -> None:
if name not in self.ATTRS:
warnings.warn(
f"Setting custom web.Application.{name} attribute "
"is discouraged",
DeprecationWarning,
stacklevel=2,
)
super().__setattr__(name, val)
# MutableMapping API
def __eq__(self, other: object) -> bool:
return self is other
@overload # type: ignore[override]
def __getitem__(self, key: AppKey[_T]) -> _T: ...
@overload
def __getitem__(self, key: str) -> Any: ...
def __getitem__(self, key: str | AppKey[_T]) -> Any:
return self._state[key]
def _check_frozen(self) -> None:
if self._frozen:
warnings.warn(
"Changing state of started or joined application is deprecated",
DeprecationWarning,
stacklevel=3,
)
@overload # type: ignore[override]
def __setitem__(self, key: AppKey[_T], value: _T) -> None: ...
@overload
def __setitem__(self, key: str, value: Any) -> None: ...
def __setitem__(self, key: str | AppKey[_T], value: Any) -> None:
self._check_frozen()
if not isinstance(key, AppKey):
warnings.warn(
"It is recommended to use web.AppKey instances for keys.\n"
+ "https://docs.aiohttp.org/en/stable/web_advanced.html"
+ "#application-s-config",
category=NotAppKeyWarning,
stacklevel=2,
)
self._state[key] = value
def __delitem__(self, key: str | AppKey[_T]) -> None:
self._check_frozen()
del self._state[key]
def __len__(self) -> int:
return len(self._state)
def __iter__(self) -> Iterator[str | AppKey[Any]]:
return iter(self._state)
def __hash__(self) -> int:
return id(self)
@overload # type: ignore[override]
def get(self, key: AppKey[_T], default: None = ...) -> _T | None: ...
@overload
def get(self, key: AppKey[_T], default: _U) -> _T | _U: ...
@overload
def get(self, key: str, default: Any = ...) -> Any: ...
def get(self, key: str | AppKey[_T], default: Any = None) -> Any:
return self._state.get(key, default)
########
@property
def loop(self) -> asyncio.AbstractEventLoop:
# Technically the loop can be None
# but we mask it by explicit type cast
# to provide more convenient type annotation
warnings.warn("loop property is deprecated", DeprecationWarning, stacklevel=2)
return cast(asyncio.AbstractEventLoop, self._loop)
def _set_loop(self, loop: asyncio.AbstractEventLoop | None) -> None:
if loop is None:
loop = asyncio.get_event_loop()
if self._loop is not None and self._loop is not loop:
raise RuntimeError(
"web.Application instance initialized with different loop"
)
self._loop = loop
# set loop debug
if self._debug is ...:
self._debug = loop.get_debug()
# set loop to sub applications
for subapp in self._subapps:
subapp._set_loop(loop)
@property
def pre_frozen(self) -> bool:
return self._pre_frozen
def pre_freeze(self) -> None:
if self._pre_frozen:
return
self._pre_frozen = True
self._middlewares.freeze()
self._router.freeze()
self._on_response_prepare.freeze()
self._cleanup_ctx.freeze()
self._on_startup.freeze()
self._on_shutdown.freeze()
self._on_cleanup.freeze()
self._middlewares_handlers = tuple(self._prepare_middleware())
self._has_legacy_middlewares = any(
not new_style for _, new_style in self._middlewares_handlers
)
# If current app and any subapp do not have middlewares avoid run all
# of the code footprint that it implies, which have a middleware
# hardcoded per app that sets up the current_app attribute. If no
# middlewares are configured the handler will receive the proper
# current_app without needing all of this code.
self._run_middlewares = True if self.middlewares else False
for subapp in self._subapps:
subapp.pre_freeze()
self._run_middlewares = self._run_middlewares or subapp._run_middlewares
@property
def frozen(self) -> bool:
return self._frozen
def freeze(self) -> None:
if self._frozen:
return
self.pre_freeze()
self._frozen = True
for subapp in self._subapps:
subapp.freeze()
@property
def debug(self) -> bool:
warnings.warn("debug property is deprecated", DeprecationWarning, stacklevel=2)
return self._debug # type: ignore[no-any-return]
def _reg_subapp_signals(self, subapp: "Application") -> None:
def reg_handler(signame: str) -> None:
subsig = getattr(subapp, signame)
async def handler(app: "Application") -> None:
await subsig.send(subapp)
appsig = getattr(self, signame)
appsig.append(handler)
reg_handler("on_startup")
reg_handler("on_shutdown")
reg_handler("on_cleanup")
def add_subapp(self, prefix: str, subapp: "Application") -> PrefixedSubAppResource:
if not isinstance(prefix, str):
raise TypeError("Prefix must be str")
prefix = prefix.rstrip("/")
if not prefix:
raise ValueError("Prefix cannot be empty")
factory = partial(PrefixedSubAppResource, prefix, subapp)
return self._add_subapp(factory, subapp)
def _add_subapp(
self, resource_factory: Callable[[], _Resource], subapp: "Application"
) -> _Resource:
if self.frozen:
raise RuntimeError("Cannot add sub application to frozen application")
if subapp.frozen:
raise RuntimeError("Cannot add frozen application")
resource = resource_factory()
self.router.register_resource(resource)
self._reg_subapp_signals(subapp)
self._subapps.append(subapp)
subapp.pre_freeze()
if self._loop is not None:
subapp._set_loop(self._loop)
return resource
def add_domain(self, domain: str, subapp: "Application") -> MatchedSubAppResource:
if not isinstance(domain, str):
raise TypeError("Domain must be str")
elif "*" in domain:
rule: Domain = MaskDomain(domain)
else:
rule = Domain(domain)
factory = partial(MatchedSubAppResource, rule, subapp)
return self._add_subapp(factory, subapp)
def add_routes(self, routes: Iterable[AbstractRouteDef]) -> list[AbstractRoute]:
return self.router.add_routes(routes)
@property
def on_response_prepare(self) -> _RespPrepareSignal:
return self._on_response_prepare
@property
def on_startup(self) -> _AppSignal:
return self._on_startup
@property
def on_shutdown(self) -> _AppSignal:
return self._on_shutdown
@property
def on_cleanup(self) -> _AppSignal:
return self._on_cleanup
@property
def cleanup_ctx(self) -> "CleanupContext":
return self._cleanup_ctx
@property
def router(self) -> UrlDispatcher:
return self._router
@property
def middlewares(self) -> _Middlewares:
return self._middlewares
def _make_handler(
self,
*,
loop: asyncio.AbstractEventLoop | None = None,
access_log_class: type[AbstractAccessLogger] = AccessLogger,
**kwargs: Any,
) -> Server:
if not issubclass(access_log_class, AbstractAccessLogger):
raise TypeError(
"access_log_class must be subclass of "
f"aiohttp.abc.AbstractAccessLogger, got {access_log_class}"
)
self._set_loop(loop)
self.freeze()
kwargs["debug"] = self._debug
kwargs["access_log_class"] = access_log_class
if self._handler_args:
for k, v in self._handler_args.items():
kwargs[k] = v
return Server(
self._handle, # type: ignore[arg-type]
request_factory=self._make_request,
loop=self._loop,
**kwargs,
)
def make_handler(
self,
*,
loop: asyncio.AbstractEventLoop | None = None,
access_log_class: type[AbstractAccessLogger] = AccessLogger,
**kwargs: Any,
) -> Server:
warnings.warn(
"Application.make_handler(...) is deprecated, use AppRunner API instead",
DeprecationWarning,
stacklevel=2,
)
return self._make_handler(
loop=loop, access_log_class=access_log_class, **kwargs
)
async def startup(self) -> None:
"""Causes on_startup signal
Should be called in the event loop along with the request handler.
"""
await self.on_startup.send(self)
async def shutdown(self) -> None:
"""Causes on_shutdown signal
Should be called before cleanup()
"""
await self.on_shutdown.send(self)
async def cleanup(self) -> None:
"""Causes on_cleanup signal
Should be called after shutdown()
"""
if self.on_cleanup.frozen:
await self.on_cleanup.send(self)
else:
# If an exception occurs in startup, ensure cleanup contexts are completed.
await self._cleanup_ctx._on_cleanup(self)
def _make_request(
self,
message: RawRequestMessage,
payload: StreamReader,
protocol: RequestHandler,
writer: AbstractStreamWriter,
task: "asyncio.Task[None]",
_cls: type[Request] = Request,
) -> Request:
if TYPE_CHECKING:
assert self._loop is not None
return _cls(
message,
payload,
protocol,
writer,
task,
self._loop,
client_max_size=self._client_max_size,
)
def _prepare_middleware(self) -> Iterator[tuple[Middleware, bool]]:
for m in reversed(self._middlewares):
if getattr(m, "__middleware_version__", None) == 1:
yield m, True
else:
warnings.warn(
f'old-style middleware "{m!r}" deprecated, see #2252',
DeprecationWarning,
stacklevel=2,
)
yield m, False
yield _fix_request_current_app(self), True
async def _handle(self, request: Request) -> StreamResponse:
loop = asyncio.get_event_loop()
debug = loop.get_debug()
match_info = await self._router.resolve(request)
if debug: # pragma: no cover
if not isinstance(match_info, AbstractMatchInfo):
raise TypeError(
"match_info should be AbstractMatchInfo "
f"instance, not {match_info!r}"
)
match_info.add_app(self)
match_info.freeze()
request._match_info = match_info
if request.headers.get(hdrs.EXPECT):
resp = await match_info.expect_handler(request)
await request.writer.drain()
if resp is not None:
return resp
handler = match_info.handler
if self._run_middlewares:
# If its a SystemRoute, don't cache building the middlewares since
# they are constructed for every MatchInfoError as a new handler
# is made each time.
if not self._has_legacy_middlewares and not isinstance(
match_info.route, SystemRoute
):
handler = _cached_build_middleware(handler, match_info.apps)
else:
for app in match_info.apps[::-1]:
for m, new_style in app._middlewares_handlers: # type: ignore[union-attr]
if new_style:
handler = update_wrapper(
partial(m, handler=handler), handler
)
else:
handler = await m(app, handler) # type: ignore[arg-type,assignment]
return await handler(request)
def __call__(self) -> "Application":
"""gunicorn compatibility"""
return self
def __repr__(self) -> str:
return f"<Application 0x{id(self):x}>"
def __bool__(self) -> bool:
return True
class CleanupError(RuntimeError):
@property
def exceptions(self) -> list[BaseException]:
return cast(list[BaseException], self.args[1])
_CleanupContextCallable = (
Callable[[Application], AbstractAsyncContextManager[None]]
| Callable[[Application], AsyncIterator[None]]
)
class CleanupContext(FrozenList[_CleanupContextCallable]):
def __init__(self) -> None:
super().__init__()
self._exits: list[AbstractAsyncContextManager[None]] = []
async def _on_startup(self, app: Application) -> None:
for cb in self:
ctx = cb(app)
if not isinstance(ctx, AbstractAsyncContextManager):
ctx = asynccontextmanager(cb)(app) # type: ignore[arg-type]
await ctx.__aenter__()
self._exits.append(ctx)
async def _on_cleanup(self, app: Application) -> None:
errors = []
for it in reversed(self._exits):
try:
await it.__aexit__(None, None, None)
except (Exception, asyncio.CancelledError) as exc:
errors.append(exc)
if errors:
if len(errors) == 1:
raise errors[0]
else:
raise CleanupError("Multiple errors on cleanup stage", errors)
+450
View File
@@ -0,0 +1,450 @@
import warnings
from typing import Any, Dict, Iterable, List, Optional, Set # noqa
from yarl import URL
from .typedefs import LooseHeaders, StrOrURL
from .web_response import Response
__all__ = (
"HTTPException",
"HTTPError",
"HTTPRedirection",
"HTTPSuccessful",
"HTTPOk",
"HTTPCreated",
"HTTPAccepted",
"HTTPNonAuthoritativeInformation",
"HTTPNoContent",
"HTTPResetContent",
"HTTPPartialContent",
"HTTPMove",
"HTTPMultipleChoices",
"HTTPMovedPermanently",
"HTTPFound",
"HTTPSeeOther",
"HTTPNotModified",
"HTTPUseProxy",
"HTTPTemporaryRedirect",
"HTTPPermanentRedirect",
"HTTPClientError",
"HTTPBadRequest",
"HTTPUnauthorized",
"HTTPPaymentRequired",
"HTTPForbidden",
"HTTPNotFound",
"HTTPMethodNotAllowed",
"HTTPNotAcceptable",
"HTTPProxyAuthenticationRequired",
"HTTPRequestTimeout",
"HTTPConflict",
"HTTPGone",
"HTTPLengthRequired",
"HTTPPreconditionFailed",
"HTTPRequestEntityTooLarge",
"HTTPRequestURITooLong",
"HTTPUnsupportedMediaType",
"HTTPRequestRangeNotSatisfiable",
"HTTPExpectationFailed",
"HTTPMisdirectedRequest",
"HTTPUnprocessableEntity",
"HTTPFailedDependency",
"HTTPUpgradeRequired",
"HTTPPreconditionRequired",
"HTTPTooManyRequests",
"HTTPRequestHeaderFieldsTooLarge",
"HTTPUnavailableForLegalReasons",
"HTTPServerError",
"HTTPInternalServerError",
"HTTPNotImplemented",
"HTTPBadGateway",
"HTTPServiceUnavailable",
"HTTPGatewayTimeout",
"HTTPVersionNotSupported",
"HTTPVariantAlsoNegotiates",
"HTTPInsufficientStorage",
"HTTPNotExtended",
"HTTPNetworkAuthenticationRequired",
)
class NotAppKeyWarning(UserWarning):
"""Warning when not using AppKey in Application."""
############################################################
# HTTP Exceptions
############################################################
class HTTPException(Response, Exception):
# You should set in subclasses:
# status = 200
status_code = -1
empty_body = False
__http_exception__ = True
def __init__(
self,
*,
headers: LooseHeaders | None = None,
reason: str | None = None,
body: Any = None,
text: str | None = None,
content_type: str | None = None,
) -> None:
if body is not None:
warnings.warn(
"body argument is deprecated for http web exceptions",
DeprecationWarning,
)
if reason is not None and ("\r" in reason or "\n" in reason):
raise ValueError("Reason cannot contain \\r or \\n")
Response.__init__(
self,
status=self.status_code,
headers=headers,
reason=reason,
body=body,
text=text,
content_type=content_type,
)
Exception.__init__(self, self.reason)
if self.body is None and not self.empty_body:
self.text = f"{self.status}: {self.reason}"
def __bool__(self) -> bool:
return True
class HTTPError(HTTPException):
"""Base class for exceptions with status codes in the 400s and 500s."""
class HTTPRedirection(HTTPException):
"""Base class for exceptions with status codes in the 300s."""
class HTTPSuccessful(HTTPException):
"""Base class for exceptions with status codes in the 200s."""
class HTTPOk(HTTPSuccessful):
status_code = 200
class HTTPCreated(HTTPSuccessful):
status_code = 201
class HTTPAccepted(HTTPSuccessful):
status_code = 202
class HTTPNonAuthoritativeInformation(HTTPSuccessful):
status_code = 203
class HTTPNoContent(HTTPSuccessful):
status_code = 204
empty_body = True
class HTTPResetContent(HTTPSuccessful):
status_code = 205
empty_body = True
class HTTPPartialContent(HTTPSuccessful):
status_code = 206
############################################################
# 3xx redirection
############################################################
class HTTPMove(HTTPRedirection):
def __init__(
self,
location: StrOrURL,
*,
headers: LooseHeaders | None = None,
reason: str | None = None,
body: Any = None,
text: str | None = None,
content_type: str | None = None,
) -> None:
if not location:
raise ValueError("HTTP redirects need a location to redirect to.")
super().__init__(
headers=headers,
reason=reason,
body=body,
text=text,
content_type=content_type,
)
self.headers["Location"] = str(URL(location))
self.location = location
class HTTPMultipleChoices(HTTPMove):
status_code = 300
class HTTPMovedPermanently(HTTPMove):
status_code = 301
class HTTPFound(HTTPMove):
status_code = 302
# This one is safe after a POST (the redirected location will be
# retrieved with GET):
class HTTPSeeOther(HTTPMove):
status_code = 303
class HTTPNotModified(HTTPRedirection):
# FIXME: this should include a date or etag header
status_code = 304
empty_body = True
class HTTPUseProxy(HTTPMove):
# Not a move, but looks a little like one
status_code = 305
class HTTPTemporaryRedirect(HTTPMove):
status_code = 307
class HTTPPermanentRedirect(HTTPMove):
status_code = 308
############################################################
# 4xx client error
############################################################
class HTTPClientError(HTTPError):
pass
class HTTPBadRequest(HTTPClientError):
status_code = 400
class HTTPUnauthorized(HTTPClientError):
status_code = 401
class HTTPPaymentRequired(HTTPClientError):
status_code = 402
class HTTPForbidden(HTTPClientError):
status_code = 403
class HTTPNotFound(HTTPClientError):
status_code = 404
class HTTPMethodNotAllowed(HTTPClientError):
status_code = 405
def __init__(
self,
method: str,
allowed_methods: Iterable[str],
*,
headers: LooseHeaders | None = None,
reason: str | None = None,
body: Any = None,
text: str | None = None,
content_type: str | None = None,
) -> None:
allow = ",".join(sorted(allowed_methods))
super().__init__(
headers=headers,
reason=reason,
body=body,
text=text,
content_type=content_type,
)
self.headers["Allow"] = allow
self.allowed_methods: set[str] = set(allowed_methods)
self.method = method.upper()
class HTTPNotAcceptable(HTTPClientError):
status_code = 406
class HTTPProxyAuthenticationRequired(HTTPClientError):
status_code = 407
class HTTPRequestTimeout(HTTPClientError):
status_code = 408
class HTTPConflict(HTTPClientError):
status_code = 409
class HTTPGone(HTTPClientError):
status_code = 410
class HTTPLengthRequired(HTTPClientError):
status_code = 411
class HTTPPreconditionFailed(HTTPClientError):
status_code = 412
class HTTPRequestEntityTooLarge(HTTPClientError):
status_code = 413
def __init__(self, max_size: float, actual_size: float = 0, **kwargs: Any) -> None:
kwargs.setdefault("text", f"Maximum request body size {max_size} exceeded.")
super().__init__(**kwargs)
class HTTPRequestURITooLong(HTTPClientError):
status_code = 414
class HTTPUnsupportedMediaType(HTTPClientError):
status_code = 415
class HTTPRequestRangeNotSatisfiable(HTTPClientError):
status_code = 416
class HTTPExpectationFailed(HTTPClientError):
status_code = 417
class HTTPMisdirectedRequest(HTTPClientError):
status_code = 421
class HTTPUnprocessableEntity(HTTPClientError):
status_code = 422
class HTTPFailedDependency(HTTPClientError):
status_code = 424
class HTTPUpgradeRequired(HTTPClientError):
status_code = 426
class HTTPPreconditionRequired(HTTPClientError):
status_code = 428
class HTTPTooManyRequests(HTTPClientError):
status_code = 429
class HTTPRequestHeaderFieldsTooLarge(HTTPClientError):
status_code = 431
class HTTPUnavailableForLegalReasons(HTTPClientError):
status_code = 451
def __init__(
self,
link: StrOrURL | None,
*,
headers: LooseHeaders | None = None,
reason: str | None = None,
body: Any = None,
text: str | None = None,
content_type: str | None = None,
) -> None:
super().__init__(
headers=headers,
reason=reason,
body=body,
text=text,
content_type=content_type,
)
self._link = None
if link:
self._link = URL(link)
self.headers["Link"] = f'<{str(self._link)}>; rel="blocked-by"'
@property
def link(self) -> URL | None:
return self._link
############################################################
# 5xx Server Error
############################################################
# Response status codes beginning with the digit "5" indicate cases in
# which the server is aware that it has erred or is incapable of
# performing the request. Except when responding to a HEAD request, the
# server SHOULD include an entity containing an explanation of the error
# situation, and whether it is a temporary or permanent condition. User
# agents SHOULD display any included entity to the user. These response
# codes are applicable to any request method.
class HTTPServerError(HTTPError):
pass
class HTTPInternalServerError(HTTPServerError):
status_code = 500
class HTTPNotImplemented(HTTPServerError):
status_code = 501
class HTTPBadGateway(HTTPServerError):
status_code = 502
class HTTPServiceUnavailable(HTTPServerError):
status_code = 503
class HTTPGatewayTimeout(HTTPServerError):
status_code = 504
class HTTPVersionNotSupported(HTTPServerError):
status_code = 505
class HTTPVariantAlsoNegotiates(HTTPServerError):
status_code = 506
class HTTPInsufficientStorage(HTTPServerError):
status_code = 507
class HTTPNotExtended(HTTPServerError):
status_code = 510
class HTTPNetworkAuthenticationRequired(HTTPServerError):
status_code = 511
+419
View File
@@ -0,0 +1,419 @@
import asyncio
import io
import os
import pathlib
import sys
from contextlib import suppress
from enum import Enum, auto
from mimetypes import MimeTypes
from stat import S_ISREG
from types import MappingProxyType
from typing import ( # noqa
IO,
TYPE_CHECKING,
Any,
Awaitable,
Callable,
Final,
Iterator,
List,
Optional,
Set,
Tuple,
Union,
cast,
)
from . import hdrs
from .abc import AbstractStreamWriter
from .helpers import DEFAULT_CHUNK_SIZE, ETAG_ANY, ETag, must_be_empty_body
from .typedefs import LooseHeaders, PathLike
from .web_exceptions import (
HTTPForbidden,
HTTPNotFound,
HTTPNotModified,
HTTPPartialContent,
HTTPPreconditionFailed,
HTTPRequestRangeNotSatisfiable,
)
from .web_response import StreamResponse
__all__ = ("FileResponse",)
if TYPE_CHECKING:
from .web_request import BaseRequest
_T_OnChunkSent = Optional[Callable[[bytes], Awaitable[None]]]
NOSENDFILE: Final[bool] = bool(os.environ.get("AIOHTTP_NOSENDFILE"))
CONTENT_TYPES: Final[MimeTypes] = MimeTypes()
# File extension to IANA encodings map that will be checked in the order defined.
ENCODING_EXTENSIONS = MappingProxyType(
{ext: CONTENT_TYPES.encodings_map[ext] for ext in (".br", ".gz")}
)
FALLBACK_CONTENT_TYPE = "application/octet-stream"
# Provide additional MIME type/extension pairs to be recognized.
# https://en.wikipedia.org/wiki/List_of_archive_formats#Compression_only
ADDITIONAL_CONTENT_TYPES = MappingProxyType(
{
"application/gzip": ".gz",
"application/x-brotli": ".br",
"application/x-bzip2": ".bz2",
"application/x-compress": ".Z",
"application/x-xz": ".xz",
}
)
class _FileResponseResult(Enum):
"""The result of the file response."""
SEND_FILE = auto() # Ie a regular file to send
NOT_ACCEPTABLE = auto() # Ie a socket, or non-regular file
PRE_CONDITION_FAILED = auto() # Ie If-Match or If-None-Match failed
NOT_MODIFIED = auto() # 304 Not Modified
# Add custom pairs and clear the encodings map so guess_type ignores them.
CONTENT_TYPES.encodings_map.clear()
for content_type, extension in ADDITIONAL_CONTENT_TYPES.items():
CONTENT_TYPES.add_type(content_type, extension)
_CLOSE_FUTURES: set[asyncio.Future[None]] = set()
class FileResponse(StreamResponse):
"""A response object can be used to send files."""
def __init__(
self,
path: PathLike,
chunk_size: int = DEFAULT_CHUNK_SIZE,
status: int = 200,
reason: str | None = None,
headers: LooseHeaders | None = None,
) -> None:
super().__init__(status=status, reason=reason, headers=headers)
self._path = pathlib.Path(path)
self._chunk_size = chunk_size
def _seek_and_read(self, fobj: IO[Any], offset: int, chunk_size: int) -> bytes:
fobj.seek(offset)
return fobj.read(chunk_size) # type: ignore[no-any-return]
async def _sendfile_fallback(
self, writer: AbstractStreamWriter, fobj: IO[Any], offset: int, count: int
) -> AbstractStreamWriter:
# To keep memory usage low,fobj is transferred in chunks
# controlled by the constructor's chunk_size argument.
chunk_size = self._chunk_size
loop = asyncio.get_event_loop()
chunk = await loop.run_in_executor(
None, self._seek_and_read, fobj, offset, min(chunk_size, count)
)
while chunk:
await writer.write(chunk)
count = count - len(chunk)
if count <= 0:
break
chunk = await loop.run_in_executor(None, fobj.read, min(chunk_size, count))
await writer.drain()
return writer
async def _sendfile(
self, request: "BaseRequest", fobj: IO[Any], offset: int, count: int
) -> AbstractStreamWriter:
writer = await super().prepare(request)
assert writer is not None
if NOSENDFILE or self.compression:
return await self._sendfile_fallback(writer, fobj, offset, count)
loop = request._loop
transport = request.transport
if transport is None:
raise ConnectionResetError("Connection lost")
try:
await loop.sendfile(transport, fobj, offset, count)
except NotImplementedError:
return await self._sendfile_fallback(writer, fobj, offset, count)
await super().write_eof()
return writer
@staticmethod
def _etag_match(etag_value: str, etags: tuple[ETag, ...], *, weak: bool) -> bool:
if len(etags) == 1 and etags[0].value == ETAG_ANY:
return True
return any(
etag.value == etag_value for etag in etags if weak or not etag.is_weak
)
async def _not_modified(
self, request: "BaseRequest", etag_value: str, last_modified: float
) -> AbstractStreamWriter | None:
self.set_status(HTTPNotModified.status_code)
self._length_check = False
self.etag = etag_value
self.last_modified = last_modified
# Delete any Content-Length headers provided by user. HTTP 304
# should always have empty response body
return await super().prepare(request)
async def _precondition_failed(
self, request: "BaseRequest"
) -> AbstractStreamWriter | None:
self.set_status(HTTPPreconditionFailed.status_code)
self.content_length = 0
return await super().prepare(request)
def _make_response(
self, request: "BaseRequest", accept_encoding: str
) -> tuple[
_FileResponseResult, io.BufferedReader | None, os.stat_result, str | None
]:
"""Return the response result, io object, stat result, and encoding.
If an uncompressed file is returned, the encoding is set to
:py:data:`None`.
This method should be called from a thread executor
since it calls os.stat which may block.
"""
file_path, st, file_encoding = self._get_file_path_stat_encoding(
accept_encoding
)
if not file_path:
return _FileResponseResult.NOT_ACCEPTABLE, None, st, None
etag_value = f"{st.st_mtime_ns:x}-{st.st_size:x}"
# https://www.rfc-editor.org/rfc/rfc9110#section-13.1.1-2
if (ifmatch := request.if_match) is not None and not self._etag_match(
etag_value, ifmatch, weak=False
):
return _FileResponseResult.PRE_CONDITION_FAILED, None, st, file_encoding
if (
(unmodsince := request.if_unmodified_since) is not None
and ifmatch is None
and st.st_mtime > unmodsince.timestamp()
):
return _FileResponseResult.PRE_CONDITION_FAILED, None, st, file_encoding
# https://www.rfc-editor.org/rfc/rfc9110#section-13.1.2-2
if (ifnonematch := request.if_none_match) is not None and self._etag_match(
etag_value, ifnonematch, weak=True
):
return _FileResponseResult.NOT_MODIFIED, None, st, file_encoding
if (
(modsince := request.if_modified_since) is not None
and ifnonematch is None
and st.st_mtime <= modsince.timestamp()
):
return _FileResponseResult.NOT_MODIFIED, None, st, file_encoding
fobj = file_path.open("rb")
with suppress(OSError):
# fstat() may not be available on all platforms
# Once we open the file, we want the fstat() to ensure
# the file has not changed between the first stat()
# and the open().
st = os.stat(fobj.fileno())
return _FileResponseResult.SEND_FILE, fobj, st, file_encoding
def _get_file_path_stat_encoding(
self, accept_encoding: str
) -> tuple[pathlib.Path | None, os.stat_result, str | None]:
file_path = self._path
for file_extension, file_encoding in ENCODING_EXTENSIONS.items():
if file_encoding not in accept_encoding:
continue
compressed_path = file_path.with_suffix(file_path.suffix + file_extension)
with suppress(OSError):
# Do not follow symlinks and ignore any non-regular files.
st = compressed_path.lstat()
if S_ISREG(st.st_mode):
return compressed_path, st, file_encoding
# Fallback to the uncompressed file
st = file_path.stat()
return file_path if S_ISREG(st.st_mode) else None, st, None
async def prepare(self, request: "BaseRequest") -> AbstractStreamWriter | None:
loop = asyncio.get_running_loop()
# Encoding comparisons should be case-insensitive
# https://www.rfc-editor.org/rfc/rfc9110#section-8.4.1
accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, "").lower()
try:
response_result, fobj, st, file_encoding = await loop.run_in_executor(
None, self._make_response, request, accept_encoding
)
except PermissionError:
self.set_status(HTTPForbidden.status_code)
return await super().prepare(request)
except OSError:
# Most likely to be FileNotFoundError or OSError for circular
# symlinks in python >= 3.13, so respond with 404.
self.set_status(HTTPNotFound.status_code)
return await super().prepare(request)
# Forbid special files like sockets, pipes, devices, etc.
if response_result is _FileResponseResult.NOT_ACCEPTABLE:
self.set_status(HTTPForbidden.status_code)
return await super().prepare(request)
if response_result is _FileResponseResult.PRE_CONDITION_FAILED:
return await self._precondition_failed(request)
if response_result is _FileResponseResult.NOT_MODIFIED:
etag_value = f"{st.st_mtime_ns:x}-{st.st_size:x}"
last_modified = st.st_mtime
return await self._not_modified(request, etag_value, last_modified)
assert fobj is not None
try:
return await self._prepare_open_file(request, fobj, st, file_encoding)
finally:
# We do not await here because we do not want to wait
# for the executor to finish before returning the response
# so the connection can begin servicing another request
# as soon as possible.
close_future = loop.run_in_executor(None, fobj.close)
# Hold a strong reference to the future to prevent it from being
# garbage collected before it completes.
_CLOSE_FUTURES.add(close_future)
close_future.add_done_callback(_CLOSE_FUTURES.remove)
async def _prepare_open_file(
self,
request: "BaseRequest",
fobj: io.BufferedReader,
st: os.stat_result,
file_encoding: str | None,
) -> AbstractStreamWriter | None:
status = self._status
file_size: int = st.st_size
file_mtime: float = st.st_mtime
count: int = file_size
start: int | None = None
if (ifrange := request.if_range) is None or file_mtime <= ifrange.timestamp():
# If-Range header check:
# condition = cached date >= last modification date
# return 206 if True else 200.
# if False:
# Range header would not be processed, return 200
# if True but Range header missing
# return 200
try:
rng = request.http_range
start = rng.start
end: int | None = rng.stop
except ValueError:
# https://tools.ietf.org/html/rfc7233:
# A server generating a 416 (Range Not Satisfiable) response to
# a byte-range request SHOULD send a Content-Range header field
# with an unsatisfied-range value.
# The complete-length in a 416 response indicates the current
# length of the selected representation.
#
# Will do the same below. Many servers ignore this and do not
# send a Content-Range header with HTTP 416
self._headers[hdrs.CONTENT_RANGE] = f"bytes */{file_size}"
self.set_status(HTTPRequestRangeNotSatisfiable.status_code)
return await super().prepare(request)
# If a range request has been made, convert start, end slice
# notation into file pointer offset and count
if start is not None:
if start < 0 and end is None: # return tail of file
start += file_size
if start < 0:
# if Range:bytes=-1000 in request header but file size
# is only 200, there would be trouble without this
start = 0
count = file_size - start
else:
# rfc7233:If the last-byte-pos value is
# absent, or if the value is greater than or equal to
# the current length of the representation data,
# the byte range is interpreted as the remainder
# of the representation (i.e., the server replaces the
# value of last-byte-pos with a value that is one less than
# the current length of the selected representation).
count = (
min(end if end is not None else file_size, file_size) - start
)
if start >= file_size:
# HTTP 416 should be returned in this case.
#
# According to https://tools.ietf.org/html/rfc7233:
# If a valid byte-range-set includes at least one
# byte-range-spec with a first-byte-pos that is less than
# the current length of the representation, or at least one
# suffix-byte-range-spec with a non-zero suffix-length,
# then the byte-range-set is satisfiable. Otherwise, the
# byte-range-set is unsatisfiable.
self._headers[hdrs.CONTENT_RANGE] = f"bytes */{file_size}"
self.set_status(HTTPRequestRangeNotSatisfiable.status_code)
return await super().prepare(request)
status = HTTPPartialContent.status_code
# Even though you are sending the whole file, you should still
# return a HTTP 206 for a Range request.
self.set_status(status)
# If the Content-Type header is not already set, guess it based on the
# extension of the request path. The encoding returned by guess_type
# can be ignored since the map was cleared above.
if hdrs.CONTENT_TYPE not in self._headers:
if sys.version_info >= (3, 13):
guesser = CONTENT_TYPES.guess_file_type
else:
guesser = CONTENT_TYPES.guess_type
self.content_type = guesser(self._path)[0] or FALLBACK_CONTENT_TYPE
if file_encoding:
self._headers[hdrs.CONTENT_ENCODING] = file_encoding
self._headers[hdrs.VARY] = hdrs.ACCEPT_ENCODING
# Disable compression if we are already sending
# a compressed file since we don't want to double
# compress.
self._compression = False
self.etag = f"{st.st_mtime_ns:x}-{st.st_size:x}"
self.last_modified = file_mtime
self.content_length = count
self._headers[hdrs.ACCEPT_RANGES] = "bytes"
if status == HTTPPartialContent.status_code:
real_start = start
assert real_start is not None
self._headers[hdrs.CONTENT_RANGE] = (
f"bytes {real_start}-{real_start + count - 1}/{file_size}"
)
# If we are sending 0 bytes calling sendfile() will throw a ValueError
if count == 0 or must_be_empty_body(request.method, status):
return await super().prepare(request)
# be aware that start could be None or int=0 here.
offset = start or 0
return await self._sendfile(request, fobj, offset, count)
+231
View File
@@ -0,0 +1,231 @@
import datetime
import functools
import logging
import os
import re
import time as time_mod
from collections.abc import Iterable
from typing import Callable, ClassVar, NamedTuple
from .abc import AbstractAccessLogger
from .web_request import BaseRequest
from .web_response import StreamResponse
class KeyMethod(NamedTuple):
key: str | tuple[str, str]
method: Callable[[BaseRequest, StreamResponse, float], str]
class AccessLogger(AbstractAccessLogger):
"""Helper object to log access.
Usage:
log = logging.getLogger("spam")
log_format = "%a %{User-Agent}i"
access_logger = AccessLogger(log, log_format)
access_logger.log(request, response, time)
Format:
%% The percent sign
%a Remote IP-address (IP-address of proxy if using reverse proxy)
%t Time when the request was started to process
%P The process ID of the child that serviced the request
%r First line of request
%s Response status code
%b Size of response in bytes, including HTTP headers
%T Time taken to serve the request, in seconds
%Tf Time taken to serve the request, in seconds with floating fraction
in .06f format
%D Time taken to serve the request, in microseconds
%{FOO}i request.headers['FOO']
%{FOO}o response.headers['FOO']
%{FOO}e os.environ['FOO']
"""
LOG_FORMAT_MAP = {
"a": "remote_address",
"t": "request_start_time",
"P": "process_id",
"r": "first_request_line",
"s": "response_status",
"b": "response_size",
"T": "request_time",
"Tf": "request_time_frac",
"D": "request_time_micro",
"i": "request_header",
"o": "response_header",
}
LOG_FORMAT = '%a %t "%r" %s %b "%{Referer}i" "%{User-Agent}i"'
FORMAT_RE = re.compile(r"%(\{([A-Za-z0-9\-_]+)\}([ioe])|[atPrsbOD]|Tf?)")
CLEANUP_RE = re.compile(r"(%[^s])")
_FORMAT_CACHE: dict[str, tuple[str, list[KeyMethod]]] = {}
_cached_tz: ClassVar[datetime.timezone | None] = None
_cached_tz_expires: ClassVar[float] = 0.0
def __init__(self, logger: logging.Logger, log_format: str = LOG_FORMAT) -> None:
"""Initialise the logger.
logger is a logger object to be used for logging.
log_format is a string with apache compatible log format description.
"""
super().__init__(logger, log_format=log_format)
_compiled_format = AccessLogger._FORMAT_CACHE.get(log_format)
if not _compiled_format:
_compiled_format = self.compile_format(log_format)
AccessLogger._FORMAT_CACHE[log_format] = _compiled_format
self._log_format, self._methods = _compiled_format
def compile_format(self, log_format: str) -> tuple[str, list[KeyMethod]]:
"""Translate log_format into form usable by modulo formatting
All known atoms will be replaced with %s
Also methods for formatting of those atoms will be added to
_methods in appropriate order
For example we have log_format = "%a %t"
This format will be translated to "%s %s"
Also contents of _methods will be
[self._format_a, self._format_t]
These method will be called and results will be passed
to translated string format.
Each _format_* method receive 'args' which is list of arguments
given to self.log
Exceptions are _format_e, _format_i and _format_o methods which
also receive key name (by functools.partial)
"""
# list of (key, method) tuples, we don't use an OrderedDict as users
# can repeat the same key more than once
methods = list()
for atom in self.FORMAT_RE.findall(log_format):
if atom[1] == "":
format_key1 = self.LOG_FORMAT_MAP[atom[0]]
m = getattr(AccessLogger, "_format_%s" % atom[0])
key_method = KeyMethod(format_key1, m)
else:
format_key2 = (self.LOG_FORMAT_MAP[atom[2]], atom[1])
m = getattr(AccessLogger, "_format_%s" % atom[2])
key_method = KeyMethod(format_key2, functools.partial(m, atom[1]))
methods.append(key_method)
log_format = self.FORMAT_RE.sub(r"%s", log_format)
log_format = self.CLEANUP_RE.sub(r"%\1", log_format)
return log_format, methods
@staticmethod
def _format_i(
key: str, request: BaseRequest, response: StreamResponse, time: float
) -> str:
if request is None:
return "(no headers)"
# suboptimal, make istr(key) once
return request.headers.get(key, "-")
@staticmethod
def _format_o(
key: str, request: BaseRequest, response: StreamResponse, time: float
) -> str:
# suboptimal, make istr(key) once
return response.headers.get(key, "-")
@staticmethod
def _format_a(request: BaseRequest, response: StreamResponse, time: float) -> str:
if request is None:
return "-"
ip = request.remote
return ip if ip is not None else "-"
@classmethod
def _get_local_time(cls) -> datetime.datetime:
if cls._cached_tz is None or time_mod.time() >= cls._cached_tz_expires:
gmtoff = time_mod.localtime().tm_gmtoff
cls._cached_tz = tz = datetime.timezone(datetime.timedelta(seconds=gmtoff))
now = datetime.datetime.now(tz)
# Expire at every 30 mins, as any DST change should occur at 0/30 mins past.
d = now + datetime.timedelta(minutes=30)
d = d.replace(minute=30 if d.minute >= 30 else 0, second=0, microsecond=0)
cls._cached_tz_expires = d.timestamp()
return now
return datetime.datetime.now(cls._cached_tz)
@staticmethod
def _format_t(request: BaseRequest, response: StreamResponse, time: float) -> str:
now = AccessLogger._get_local_time()
start_time = now - datetime.timedelta(seconds=time)
return start_time.strftime("[%d/%b/%Y:%H:%M:%S %z]")
@staticmethod
def _format_P(request: BaseRequest, response: StreamResponse, time: float) -> str:
return "<%s>" % os.getpid()
@staticmethod
def _format_r(request: BaseRequest, response: StreamResponse, time: float) -> str:
if request is None:
return "-"
return f"{request.method} {request.path_qs} HTTP/{request.version.major}.{request.version.minor}"
@staticmethod
def _format_s(request: BaseRequest, response: StreamResponse, time: float) -> int:
return response.status
@staticmethod
def _format_b(request: BaseRequest, response: StreamResponse, time: float) -> int:
return response.body_length
@staticmethod
def _format_T(request: BaseRequest, response: StreamResponse, time: float) -> str:
return str(round(time))
@staticmethod
def _format_Tf(request: BaseRequest, response: StreamResponse, time: float) -> str:
return "%06f" % time
@staticmethod
def _format_D(request: BaseRequest, response: StreamResponse, time: float) -> str:
return str(round(time * 1000000))
def _format_line(
self, request: BaseRequest, response: StreamResponse, time: float
) -> Iterable[tuple[str | tuple[str, str], str]]:
return [(key, method(request, response, time)) for key, method in self._methods]
@property
def enabled(self) -> bool:
"""Check if logger is enabled."""
# Avoid formatting the log line if it will not be emitted.
return self.logger.isEnabledFor(logging.INFO)
def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
try:
fmt_info = self._format_line(request, response, time)
values = list()
extra: dict[str, str | dict[str, str]] = dict()
for key, value in fmt_info:
values.append(value)
if isinstance(key, str):
extra[key] = value
else:
k1, k2 = key
dct: dict[str, str] = extra.get(k1, {}) # type: ignore[assignment]
dct[k2] = value
extra[k1] = dct
self.logger.info(self._log_format % tuple(values), extra=extra)
except Exception:
self.logger.exception("Error in logging")
+121
View File
@@ -0,0 +1,121 @@
import re
from typing import TYPE_CHECKING, TypeVar
from .typedefs import Handler, Middleware
from .web_exceptions import HTTPMove, HTTPPermanentRedirect
from .web_request import Request
from .web_response import StreamResponse
from .web_urldispatcher import SystemRoute
__all__ = (
"middleware",
"normalize_path_middleware",
)
if TYPE_CHECKING:
from .web_app import Application
_Func = TypeVar("_Func")
async def _check_request_resolves(request: Request, path: str) -> tuple[bool, Request]:
alt_request = request.clone(rel_url=path)
match_info = await request.app.router.resolve(alt_request)
alt_request._match_info = match_info
if match_info.http_exception is None:
return True, alt_request
return False, request
def middleware(f: _Func) -> _Func:
f.__middleware_version__ = 1 # type: ignore[attr-defined]
return f
def normalize_path_middleware(
*,
append_slash: bool = True,
remove_slash: bool = False,
merge_slashes: bool = True,
redirect_class: type[HTTPMove] = HTTPPermanentRedirect,
) -> Middleware:
"""Factory for producing a middleware that normalizes the path of a request.
Normalizing means:
- Add or remove a trailing slash to the path.
- Double slashes are replaced by one.
The middleware returns as soon as it finds a path that resolves
correctly. The order if both merge and append/remove are enabled is
1) merge slashes
2) append/remove slash
3) both merge slashes and append/remove slash.
If the path resolves with at least one of those conditions, it will
redirect to the new path.
Only one of `append_slash` and `remove_slash` can be enabled. If both
are `True` the factory will raise an assertion error
If `append_slash` is `True` the middleware will append a slash when
needed. If a resource is defined with trailing slash and the request
comes without it, it will append it automatically.
If `remove_slash` is `True`, `append_slash` must be `False`. When enabled
the middleware will remove trailing slashes and redirect if the resource
is defined
If merge_slashes is True, merge multiple consecutive slashes in the
path into one.
"""
correct_configuration = not (append_slash and remove_slash)
assert correct_configuration, "Cannot both remove and append slash"
@middleware
async def impl(request: Request, handler: Handler) -> StreamResponse:
if isinstance(request.match_info.route, SystemRoute):
paths_to_check = []
if "?" in request.raw_path:
path, query = request.raw_path.split("?", 1)
query = "?" + query
else:
query = ""
path = request.raw_path
if merge_slashes:
paths_to_check.append(re.sub("//+", "/", path))
if append_slash and not request.path.endswith("/"):
paths_to_check.append(path + "/")
if remove_slash and request.path.endswith("/"):
paths_to_check.append(path[:-1])
if merge_slashes and append_slash:
paths_to_check.append(re.sub("//+", "/", path + "/"))
if merge_slashes and remove_slash:
merged_slashes = re.sub("//+", "/", path)
paths_to_check.append(merged_slashes[:-1])
for path in paths_to_check:
path = re.sub("^//+", "/", path) # SECURITY: GHSA-v6wp-4m6f-gcjg
resolves, request = await _check_request_resolves(request, path)
if resolves:
raise redirect_class(request.raw_path + query)
return await handler(request)
return impl
def _fix_request_current_app(app: "Application") -> Middleware:
@middleware
async def impl(request: Request, handler: Handler) -> StreamResponse:
match_info = request.match_info
prev = match_info.current_app
match_info.current_app = app
try:
return await handler(request)
finally:
match_info.current_app = prev
return impl
+874
View File
@@ -0,0 +1,874 @@
import asyncio
import asyncio.streams
import sys
import traceback
import warnings
from collections import deque
from collections.abc import Awaitable, Callable, Sequence
from contextlib import suppress
from html import escape as html_escape
from http import HTTPStatus
from logging import Logger
from typing import TYPE_CHECKING, Any, Optional, cast
import attr
import yarl
from propcache import under_cached_property
from .abc import AbstractAccessLogger, AbstractStreamWriter
from .base_protocol import PAUSE_RESUME_READING_ERRORS, BaseProtocol
from .helpers import DEFAULT_CHUNK_SIZE, ceil_timeout
from .http import (
HttpProcessingError,
HttpRequestParser,
HttpVersion10,
RawRequestMessage,
StreamWriter,
WebSocketReader,
)
from .http_exceptions import BadHttpMethod
from .log import access_logger, server_logger
from .streams import EMPTY_PAYLOAD, StreamReader
from .tcp_helpers import tcp_keepalive
from .web_exceptions import HTTPException, HTTPInternalServerError
from .web_log import AccessLogger
from .web_request import BaseRequest
from .web_response import Response, StreamResponse
__all__ = ("RequestHandler", "RequestPayloadError", "PayloadAccessError")
# Max parsed-but-unhandled pipelined requests buffered per connection before
# reading is paused. Bounds memory a client can pin by keeping one handler busy
# and pipelining behind it; reading resumes as the queue drains.
MAX_MSG_QUEUE_SIZE = 32
if TYPE_CHECKING:
import ssl
from .web_server import Server
_RequestFactory = Callable[
[
RawRequestMessage,
StreamReader,
"RequestHandler",
AbstractStreamWriter,
"asyncio.Task[None]",
],
BaseRequest,
]
_RequestHandler = Callable[[BaseRequest], Awaitable[StreamResponse]]
ERROR = RawRequestMessage(
"UNKNOWN",
"/",
HttpVersion10,
{}, # type: ignore[arg-type]
{}, # type: ignore[arg-type]
True,
None,
False,
False,
yarl.URL("/"),
)
class RequestPayloadError(Exception):
"""Payload parsing error."""
class PayloadAccessError(Exception):
"""Payload was accessed after response was sent."""
_PAYLOAD_ACCESS_ERROR = PayloadAccessError()
@attr.s(auto_attribs=True, frozen=True, slots=True)
class _ErrInfo:
status: int
exc: BaseException
message: str
_MsgType = tuple[RawRequestMessage | _ErrInfo, StreamReader]
class RequestHandler(BaseProtocol):
"""HTTP protocol implementation.
RequestHandler handles incoming HTTP request. It reads request line,
request headers and request payload and calls handle_request() method.
By default it always returns with 404 response.
RequestHandler handles errors in incoming request, like bad
status line, bad headers or incomplete payload. If any error occurs,
connection gets closed.
keepalive_timeout -- number of seconds before closing
keep-alive connection
tcp_keepalive -- TCP keep-alive is on, default is on
debug -- enable debug mode
logger -- custom logger object
access_log_class -- custom class for access_logger
access_log -- custom logging object
access_log_format -- access log format string
loop -- Optional event loop
max_line_size -- Optional maximum header line size
max_field_size -- Optional maximum header field size
max_headers -- Optional maximum header size
timeout_ceil_threshold -- Optional value to specify
threshold to ceil() timeout
values
"""
__slots__ = (
"max_field_size",
"max_headers",
"max_line_size",
"_request_count",
"_keepalive",
"_manager",
"_request_handler",
"_request_factory",
"_tcp_keepalive",
"_next_keepalive_close_time",
"_keepalive_handle",
"_keepalive_timeout",
"_lingering_time",
"_messages",
"_max_msg_queue_size",
"_msg_queue_resume_size",
"_msg_queue_paused",
"_message_tail",
"_handler_waiter",
"_waiter",
"_task_handler",
"_payload_parser",
"_data_received_cb",
"logger",
"debug",
"access_log",
"access_logger",
"_close",
"_force_close",
"_current_request",
"_timeout_ceil_threshold",
"_request_in_progress",
"_logging_enabled",
"_cache",
)
def __init__(
self,
manager: "Server",
*,
loop: asyncio.AbstractEventLoop,
# Default should be high enough that it's likely longer than a reverse proxy.
keepalive_timeout: float = 3630,
tcp_keepalive: bool = True,
logger: Logger = server_logger,
access_log_class: type[AbstractAccessLogger] = AccessLogger,
access_log: Logger = access_logger,
access_log_format: str = AccessLogger.LOG_FORMAT,
debug: bool = False,
max_line_size: int = 8190,
max_headers: int = 128,
max_field_size: int = 8190,
lingering_time: float = 10.0,
read_bufsize: int = DEFAULT_CHUNK_SIZE,
auto_decompress: bool = True,
timeout_ceil_threshold: float = 5,
):
self._max_msg_queue_size = MAX_MSG_QUEUE_SIZE
# Low-water mark: resume reading once the queue drains to half the limit
# so we refill in batches instead of churning pause/resume per request.
self._msg_queue_resume_size = MAX_MSG_QUEUE_SIZE // 2
# Set before super().__init__ so _reading_paused_for_msg_queue() is safe
# if BaseProtocol ever triggers a resume during init.
self._msg_queue_paused = False
parser = HttpRequestParser(
self,
loop,
read_bufsize,
max_line_size=max_line_size,
max_field_size=max_field_size,
max_headers=max_headers,
payload_exception=RequestPayloadError,
auto_decompress=auto_decompress,
max_msg_queue_size=MAX_MSG_QUEUE_SIZE,
)
super().__init__(loop, parser)
# _request_count is the number of requests processed with the same connection.
self._request_count = 0
self._keepalive = False
self._current_request: BaseRequest | None = None
self._manager: Server | None = manager
self._request_handler: _RequestHandler | None = manager.request_handler
self._request_factory: _RequestFactory | None = manager.request_factory
self.max_line_size = max_line_size
self.max_headers = max_headers
self.max_field_size = max_field_size
self._tcp_keepalive = tcp_keepalive
# placeholder to be replaced on keepalive timeout setup
self._next_keepalive_close_time = 0.0
self._keepalive_handle: asyncio.Handle | None = None
self._keepalive_timeout = keepalive_timeout
self._lingering_time = float(lingering_time)
self._messages: deque[_MsgType] = deque()
self._message_tail = b""
self._data_received_cb: Callable[[], None] | None = None
self._waiter: asyncio.Future[None] | None = None
self._handler_waiter: asyncio.Future[None] | None = None
self._task_handler: asyncio.Task[None] | None = None
self._payload_parser: Any = None
self._timeout_ceil_threshold: float = 5
try:
self._timeout_ceil_threshold = float(timeout_ceil_threshold)
except (TypeError, ValueError):
pass
self.logger = logger
self.debug = debug
self.access_log = access_log
if access_log:
self.access_logger: AbstractAccessLogger | None = access_log_class(
access_log, access_log_format
)
self._logging_enabled = self.access_logger.enabled
else:
self.access_logger = None
self._logging_enabled = False
self._close = False
self._force_close = False
self._request_in_progress = False
self._cache: dict[str, Any] = {}
def __repr__(self) -> str:
return "<{} {}>".format(
self.__class__.__name__,
"connected" if self.transport is not None else "disconnected",
)
@under_cached_property
def ssl_context(self) -> Optional["ssl.SSLContext"]:
"""Return SSLContext if available."""
return (
None
if self.transport is None
else self.transport.get_extra_info("sslcontext")
)
@under_cached_property
def peername(
self,
) -> str | tuple[str, int, int, int] | tuple[str, int] | None:
"""Return peername if available."""
return (
None
if self.transport is None
else self.transport.get_extra_info("peername")
)
@under_cached_property
def sockname(
self,
) -> str | tuple[str, int, int, int] | tuple[str, int] | None:
"""Return sockname if available."""
return (
None
if self.transport is None
else self.transport.get_extra_info("sockname")
)
@property
def keepalive_timeout(self) -> float:
return self._keepalive_timeout
async def shutdown(self, timeout: float | None = 15.0) -> None:
"""Do worker process exit preparations.
We need to clean up everything and stop accepting requests.
It is especially important for keep-alive connections.
"""
self._force_close = True
if self._keepalive_handle is not None:
self._keepalive_handle.cancel()
# Wait for graceful handler completion
if self._request_in_progress:
# The future is only created when we are shutting
# down while the handler is still processing a request
# to avoid creating a future for every request.
self._handler_waiter = self._loop.create_future()
try:
async with ceil_timeout(timeout):
await self._handler_waiter
except (asyncio.CancelledError, asyncio.TimeoutError):
self._handler_waiter = None
if (
sys.version_info >= (3, 11)
and (task := asyncio.current_task())
and task.cancelling()
):
raise
# Then cancel handler and wait
try:
async with ceil_timeout(timeout):
if self._current_request is not None:
self._current_request._cancel(asyncio.CancelledError())
if self._task_handler is not None and not self._task_handler.done():
await asyncio.shield(self._task_handler)
except (asyncio.CancelledError, asyncio.TimeoutError):
if (
sys.version_info >= (3, 11)
and (task := asyncio.current_task())
and task.cancelling()
):
raise
# force-close non-idle handler
if self._task_handler is not None:
self._task_handler.cancel()
self.force_close()
def connection_made(self, transport: asyncio.BaseTransport) -> None:
super().connection_made(transport)
real_transport = cast(asyncio.Transport, transport)
if self._tcp_keepalive:
tcp_keepalive(real_transport)
assert self._manager is not None
self._manager.connection_made(self, real_transport)
loop = self._loop
if sys.version_info >= (3, 12):
task = asyncio.Task(self.start(), loop=loop, eager_start=True)
else:
task = loop.create_task(self.start())
self._task_handler = task
def connection_lost(self, exc: BaseException | None) -> None:
if self._manager is None:
return
self._manager.connection_lost(self, exc)
# Grab value before setting _manager to None.
handler_cancellation = self._manager.handler_cancellation
self.force_close()
super().connection_lost(exc)
self._manager = None
self._request_factory = None
self._request_handler = None
self._parser = None
if self._keepalive_handle is not None:
self._keepalive_handle.cancel()
if self._current_request is not None:
if exc is None:
exc = ConnectionResetError("Connection lost")
self._current_request._cancel(exc)
if handler_cancellation and self._task_handler is not None:
self._task_handler.cancel()
self._task_handler = None
if self._payload_parser is not None:
self._payload_parser.feed_eof()
self._payload_parser = None
def set_parser(
self,
parser: WebSocketReader,
data_received_cb: Callable[[], None] | None = None,
) -> None:
assert self._payload_parser is None
self._payload_parser = parser
self._data_received_cb = data_received_cb
if self._message_tail:
self._payload_parser.feed_data(self._message_tail)
self._message_tail = b""
def eof_received(self) -> None:
pass
def data_received(self, data: bytes) -> None:
if self._force_close or self._close:
return
# parse http messages
messages: Sequence[_MsgType]
if self._payload_parser is None and not self._upgraded:
assert self._parser is not None
try:
messages, upgraded, tail = self._parser.feed_data(data)
except HttpProcessingError as exc:
messages = [
(_ErrInfo(status=400, exc=exc, message=exc.message), EMPTY_PAYLOAD)
]
upgraded = False
tail = b""
for msg, payload in messages:
self._request_count += 1
self._messages.append((msg, payload))
waiter = self._waiter
if messages and waiter is not None and not waiter.done():
# don't set result twice
waiter.set_result(None)
# Queue full: pause the transport (the parser already stopped
# emitting). start() resumes as it drains the queue.
if (
not self._msg_queue_paused
and len(self._messages) >= self._max_msg_queue_size
):
self._pause_msg_queue_reading()
self._upgraded = upgraded
if upgraded and tail:
self._message_tail = tail
# no parser, just store
elif self._payload_parser is None and self._upgraded and data:
self._message_tail += data
# feed payload
elif data:
if self._data_received_cb is not None:
self._data_received_cb()
eof, tail = self._payload_parser.feed_data(data)
if eof:
self.close()
def _reading_paused_for_msg_queue(self) -> bool:
return self._msg_queue_paused
def _pause_msg_queue_reading(self) -> None:
self._msg_queue_paused = True
if self.transport is not None:
try:
self.transport.pause_reading()
except PAUSE_RESUME_READING_ERRORS:
# Transport lacks flow control; nothing to pause. Intentionally
# ignored (see PAUSE_RESUME_READING_ERRORS; do not use suppress).
pass
def _resume_msg_queue_reading(self) -> None:
if not self._upgraded:
# Reparse buffered pipelined requests while still marked paused so
# a refill past the limit does not re-pause an already-paused
# transport; only resume below once it stayed under the limit.
self.data_received(b"")
if len(self._messages) >= self._max_msg_queue_size:
return
self._msg_queue_paused = False
if not self._reading_paused and self.transport is not None:
try:
self.transport.resume_reading()
except PAUSE_RESUME_READING_ERRORS:
# Transport lacks flow control; nothing to resume. Intentionally
# ignored (see PAUSE_RESUME_READING_ERRORS; do not use suppress).
pass
def keep_alive(self, val: bool) -> None:
"""Set keep-alive connection mode.
:param bool val: new state.
"""
self._keepalive = val
if self._keepalive_handle:
self._keepalive_handle.cancel()
self._keepalive_handle = None
def close(self) -> None:
"""Close connection.
Stop accepting new pipelining messages and close
connection when handlers done processing messages.
"""
self._close = True
if self._waiter:
self._waiter.cancel()
def force_close(self) -> None:
"""Forcefully close connection."""
self._force_close = True
if self._waiter:
self._waiter.cancel()
if self.transport is not None:
self.transport.close()
self.transport = None
def log_access(
self, request: BaseRequest, response: StreamResponse, time: float | None
) -> None:
if self._logging_enabled and self.access_logger is not None:
if TYPE_CHECKING:
assert time is not None
self.access_logger.log(request, response, self._loop.time() - time)
def log_debug(self, *args: Any, **kw: Any) -> None:
if self.debug:
self.logger.debug(*args, **kw)
def log_exception(self, *args: Any, **kw: Any) -> None:
self.logger.exception(*args, **kw)
def _process_keepalive(self) -> None:
self._keepalive_handle = None
if self._force_close or not self._keepalive:
return
loop = self._loop
now = loop.time()
close_time = self._next_keepalive_close_time
if now < close_time:
# Keep alive close check fired too early, reschedule
self._keepalive_handle = loop.call_at(close_time, self._process_keepalive)
return
# handler in idle state
if self._waiter and not self._waiter.done():
self.force_close()
async def _handle_request(
self,
request: BaseRequest,
start_time: float | None,
request_handler: Callable[[BaseRequest], Awaitable[StreamResponse]],
) -> tuple[StreamResponse, bool]:
self._request_in_progress = True
try:
try:
self._current_request = request
resp = await request_handler(request)
finally:
self._current_request = None
except HTTPException as exc:
resp = exc
resp, reset = await self.finish_response(request, resp, start_time)
except asyncio.CancelledError:
raise
except asyncio.TimeoutError as exc:
self.log_debug("Request handler timed out.", exc_info=exc)
resp = self.handle_error(request, 504)
resp, reset = await self.finish_response(request, resp, start_time)
except Exception as exc:
resp = self.handle_error(request, 500, exc)
resp, reset = await self.finish_response(request, resp, start_time)
else:
# Deprecation warning (See #2415)
if getattr(resp, "__http_exception__", False):
warnings.warn(
"returning HTTPException object is deprecated "
"(#2415) and will be removed, "
"please raise the exception instead",
DeprecationWarning,
)
resp, reset = await self.finish_response(request, resp, start_time)
finally:
self._request_in_progress = False
if self._handler_waiter is not None:
self._handler_waiter.set_result(None)
return resp, reset
async def start(self) -> None:
"""Process incoming request.
It reads request line, request headers and request payload, then
calls handle_request() method. Subclass has to override
handle_request(). start() handles various exceptions in request
or response handling. Connection is being closed always unless
keep_alive(True) specified.
"""
loop = self._loop
manager = self._manager
assert manager is not None
keepalive_timeout = self._keepalive_timeout
resp = None
assert self._request_factory is not None
assert self._request_handler is not None
while not self._force_close:
if not self._messages:
try:
# wait for next request
self._waiter = loop.create_future()
await self._waiter
finally:
self._waiter = None
message, payload = self._messages.popleft()
# Free a parser slot; resume reading once drained to low water so
# pipelining keeps flowing while this request is handled.
# no branch: _parser is only None after connection_lost, whose path
# exits this loop, so the None case is not reachably exercisable.
if self._parser is not None: # pragma: no branch
self._parser.message_consumed()
if (
self._msg_queue_paused
and len(self._messages) <= self._msg_queue_resume_size
):
self._resume_msg_queue_reading()
# time is only fetched if logging is enabled as otherwise
# its thrown away and never used.
start = loop.time() if self._logging_enabled else None
manager.requests_count += 1
writer = StreamWriter(self, loop)
if isinstance(message, _ErrInfo):
# make request_factory work
request_handler = self._make_error_handler(message)
message = ERROR
else:
request_handler = self._request_handler
# Important don't hold a reference to the current task
# as on traceback it will prevent the task from being
# collected and will cause a memory leak.
request = self._request_factory(
message,
payload,
self,
writer,
self._task_handler or asyncio.current_task(loop), # type: ignore[arg-type]
)
try:
# a new task is used for copy context vars (#3406)
coro = self._handle_request(request, start, request_handler)
if sys.version_info >= (3, 12):
task = asyncio.Task(coro, loop=loop, eager_start=True)
else:
task = loop.create_task(coro)
try:
resp, reset = await task
except ConnectionError:
self.log_debug("Ignored premature client disconnection")
break
# Drop the processed task from asyncio.Task.all_tasks() early
del task
if reset:
self.log_debug("Ignored premature client disconnection 2")
break
# notify server about keep-alive
self._keepalive = bool(resp.keep_alive)
# check payload
if not payload.is_eof():
lingering_time = self._lingering_time
if not self._force_close and lingering_time:
self.log_debug(
"Start lingering close timer for %s sec.", lingering_time
)
now = loop.time()
end_t = now + lingering_time
try:
while not payload.is_eof() and now < end_t:
async with ceil_timeout(end_t - now):
# read and ignore
await payload.readany()
now = loop.time()
except (asyncio.CancelledError, asyncio.TimeoutError):
if (
sys.version_info >= (3, 11)
and (t := asyncio.current_task())
and t.cancelling()
):
raise
# if payload still uncompleted
if not payload.is_eof() and not self._force_close:
self.log_debug("Uncompleted request.")
self.close()
payload.set_exception(_PAYLOAD_ACCESS_ERROR)
except asyncio.CancelledError:
self.log_debug("Ignored premature client disconnection")
self.force_close()
raise
except Exception as exc:
self.log_exception("Unhandled exception", exc_info=exc)
self.force_close()
except BaseException:
self.force_close()
raise
finally:
request._task = None # type: ignore[assignment] # Break reference cycle in case of exception
if self.transport is None and resp is not None:
self.log_debug("Ignored premature client disconnection.")
if self._keepalive and not self._close and not self._force_close:
# start keep-alive timer
close_time = loop.time() + keepalive_timeout
self._next_keepalive_close_time = close_time
if self._keepalive_handle is None:
self._keepalive_handle = loop.call_at(
close_time, self._process_keepalive
)
else:
break
# remove handler, close transport if no handlers left
if not self._force_close:
self._task_handler = None
if self.transport is not None:
self.transport.close()
async def finish_response(
self, request: BaseRequest, resp: StreamResponse, start_time: float | None
) -> tuple[StreamResponse, bool]:
"""Prepare the response and write_eof, then log access.
This has to
be called within the context of any exception so the access logger
can get exception information. Returns True if the client disconnects
prematurely.
"""
request._finish()
if self._parser is not None:
self._parser.set_upgraded(False)
self._upgraded = False
if self._message_tail:
messages, _upgraded, tail = self._parser.feed_data(self._message_tail)
self._message_tail = tail
for msg, payload in messages:
self._request_count += 1
self._messages.append((msg, payload))
# This shouldn't be possible. If a future refactor results in this
# failing, then the code may need to be updated to set the waiter.
assert self._waiter is None
try:
prepare_meth = resp.prepare
except AttributeError:
if resp is None:
self.log_exception("Missing return statement on request handler")
else:
self.log_exception(
f"Web-handler should return a response instance, got {resp!r}"
)
exc = HTTPInternalServerError()
resp = Response(
status=exc.status, reason=exc.reason, text=exc.text, headers=exc.headers
)
prepare_meth = resp.prepare
try:
await prepare_meth(request)
await resp.write_eof()
except ConnectionError:
self.log_access(request, resp, start_time)
return resp, True
self.log_access(request, resp, start_time)
return resp, False
def handle_error(
self,
request: BaseRequest,
status: int = 500,
exc: BaseException | None = None,
message: str | None = None,
) -> StreamResponse:
"""Handle errors.
Returns HTTP response with specific status code. Logs additional
information. It always closes current connection.
"""
if self._request_count == 1 and isinstance(exc, BadHttpMethod):
# BadHttpMethod is common when a client sends non-HTTP
# or encrypted traffic to an HTTP port. This is expected
# to happen when connected to the public internet so we log
# it at the debug level as to not fill logs with noise.
self.logger.debug(
"Error handling request from %s", request.remote, exc_info=exc
)
else:
self.log_exception(
"Error handling request from %s", request.remote, exc_info=exc
)
# some data already got sent, connection is broken
if request.writer.output_size > 0:
raise ConnectionError(
"Response is sent already, cannot send another response "
"with the error message"
)
ct = "text/plain"
if status == HTTPStatus.INTERNAL_SERVER_ERROR:
title = f"{HTTPStatus.INTERNAL_SERVER_ERROR.value} {HTTPStatus.INTERNAL_SERVER_ERROR.phrase}"
msg = HTTPStatus.INTERNAL_SERVER_ERROR.description
tb = None
if self.debug:
with suppress(Exception):
tb = traceback.format_exc()
if "text/html" in request.headers.get("Accept", ""):
if tb:
tb = html_escape(tb)
msg = f"<h2>Traceback:</h2>\n<pre>{tb}</pre>"
message = (
"<html><head>"
f"<title>{title}</title>"
f"</head><body>\n<h1>{title}</h1>"
f"\n{msg}\n</body></html>\n"
)
ct = "text/html"
else:
if tb:
msg = tb
message = title + "\n\n" + msg
resp = Response(status=status, text=message, content_type=ct)
resp.force_close()
return resp
def _make_error_handler(
self, err_info: _ErrInfo
) -> Callable[[BaseRequest], Awaitable[StreamResponse]]:
async def handler(request: BaseRequest) -> StreamResponse:
return self.handle_error(
request, err_info.status, err_info.exc, err_info.message
)
return handler
+949
View File
@@ -0,0 +1,949 @@
import asyncio
import datetime
import io
import re
import string
import tempfile
import types
import warnings
from collections.abc import Iterator, Mapping, MutableMapping
from re import Pattern
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar, cast, overload
from urllib.parse import parse_qsl
import attr
from multidict import (
CIMultiDict,
CIMultiDictProxy,
MultiDict,
MultiDictProxy,
MultiMapping,
)
from yarl import URL
from . import hdrs
from ._cookie_helpers import parse_cookie_header
from .abc import AbstractStreamWriter
from .helpers import (
_SENTINEL,
DEBUG,
DEFAULT_CHUNK_SIZE,
ETAG_ANY,
LIST_QUOTED_ETAG_RE,
ChainMapProxy,
ETag,
HeadersMixin,
RequestKey,
parse_http_date,
reify,
sentinel,
set_exception,
)
from .http_parser import RawRequestMessage
from .http_writer import HttpVersion
from .multipart import BodyPartReader, MultipartReader
from .streams import EmptyStreamReader, StreamReader
from .typedefs import (
DEFAULT_JSON_DECODER,
JSONDecoder,
LooseHeaders,
RawHeaders,
StrOrURL,
)
from .web_exceptions import HTTPRequestEntityTooLarge, NotAppKeyWarning
from .web_response import StreamResponse
__all__ = ("BaseRequest", "FileField", "Request")
if TYPE_CHECKING:
from .web_app import Application
from .web_protocol import RequestHandler
from .web_urldispatcher import UrlMappingMatchInfo
_T = TypeVar("_T")
@attr.s(auto_attribs=True, frozen=True, slots=True)
class FileField:
name: str
filename: str
file: io.BufferedReader
content_type: str
headers: CIMultiDictProxy[str]
_Post = str | bytes | bytearray | FileField
_TCHAR: Final[str] = string.digits + string.ascii_letters + r"!#$%&'*+.^_`|~-"
# '-' at the end to prevent interpretation as range in a char class
_TOKEN: Final[str] = rf"[{_TCHAR}]+"
_QDTEXT: Final[str] = r"[{}]".format(
r"".join(chr(c) for c in (0x09, 0x20, 0x21) + tuple(range(0x23, 0x7F)))
)
# qdtext includes 0x5C to escape 0x5D ('\]')
# qdtext excludes obs-text (because obsoleted, and encoding not specified)
_QUOTED_PAIR: Final[str] = r"\\[\t !-~]"
_QUOTED_STRING: Final[str] = rf'"(?:{_QUOTED_PAIR}|{_QDTEXT})*"'
# This does not have a ReDOS/performance concern as long as it used with re.match().
_FORWARDED_PAIR: Final[str] = rf"({_TOKEN})=({_TOKEN}|{_QUOTED_STRING})(:\d{{1,4}})?"
_QUOTED_PAIR_REPLACE_RE: Final[Pattern[str]] = re.compile(r"\\([\t !-~])")
# same pattern as _QUOTED_PAIR but contains a capture group
_FORWARDED_PAIR_RE: Final[Pattern[str]] = re.compile(_FORWARDED_PAIR)
############################################################
# HTTP Request
############################################################
class BaseRequest(MutableMapping[str | RequestKey[Any], Any], HeadersMixin):
POST_METHODS = {
hdrs.METH_PATCH,
hdrs.METH_POST,
hdrs.METH_PUT,
hdrs.METH_TRACE,
hdrs.METH_DELETE,
}
ATTRS = HeadersMixin.ATTRS | frozenset(
[
"_message",
"_protocol",
"_payload_writer",
"_payload",
"_headers",
"_method",
"_version",
"_rel_url",
"_post",
"_read_bytes",
"_state",
"_cache",
"_task",
"_client_max_size",
"_loop",
"_transport_sslcontext",
"_transport_peername",
"_transport_sockname",
]
)
_post: MultiDictProxy[_Post] | None = None
_read_bytes: bytes | None = None
_seen_str_keys: set[str] = set()
def __init__(
self,
message: RawRequestMessage,
payload: StreamReader,
protocol: "RequestHandler",
payload_writer: AbstractStreamWriter,
task: "asyncio.Task[None]",
loop: asyncio.AbstractEventLoop,
*,
client_max_size: int = 1024**2,
state: dict[RequestKey[Any] | str, Any] | None = None,
scheme: str | None = None,
host: str | None = None,
remote: str | None = None,
) -> None:
self._message = message
self._protocol = protocol
self._payload_writer = payload_writer
self._payload = payload
self._headers: CIMultiDictProxy[str] = message.headers
self._method = message.method
self._version = message.version
self._cache: dict[str, Any] = {}
url = message.url
if url.absolute:
if scheme is not None:
url = url.with_scheme(scheme)
if host is not None:
url = url.with_host(host)
# absolute URL is given,
# override auto-calculating url, host, and scheme
# all other properties should be good
self._cache["url"] = url
self._cache["host"] = url.host
self._cache["scheme"] = url.scheme
self._rel_url = url.relative()
else:
self._rel_url = url
if scheme is not None:
self._cache["scheme"] = scheme
if host is not None:
self._cache["host"] = host
self._state = {} if state is None else state
self._task = task
self._client_max_size = client_max_size
self._loop = loop
self._transport_sslcontext = protocol.ssl_context
self._transport_peername = protocol.peername
self._transport_sockname = protocol.sockname
if remote is not None:
self._cache["remote"] = remote
def clone(
self,
*,
method: str | _SENTINEL = sentinel,
rel_url: StrOrURL | _SENTINEL = sentinel,
headers: LooseHeaders | _SENTINEL = sentinel,
scheme: str | _SENTINEL = sentinel,
host: str | _SENTINEL = sentinel,
remote: str | _SENTINEL = sentinel,
client_max_size: int | _SENTINEL = sentinel,
) -> "BaseRequest":
"""Clone itself with replacement some attributes.
Creates and returns a new instance of Request object. If no parameters
are given, an exact copy is returned. If a parameter is not passed, it
will reuse the one from the current request object.
"""
if self._read_bytes:
raise RuntimeError("Cannot clone request after reading its content")
dct: dict[str, Any] = {}
if method is not sentinel:
dct["method"] = method
if rel_url is not sentinel:
new_url: URL = URL(rel_url)
dct["url"] = new_url
dct["path"] = str(new_url)
if headers is not sentinel:
# a copy semantic
dct["headers"] = CIMultiDictProxy(CIMultiDict(headers))
dct["raw_headers"] = tuple(
(k.encode("utf-8"), v.encode("utf-8"))
for k, v in dct["headers"].items()
)
message = self._message._replace(**dct)
kwargs = {}
if scheme is not sentinel:
kwargs["scheme"] = scheme
if host is not sentinel:
kwargs["host"] = host
if remote is not sentinel:
kwargs["remote"] = remote
if client_max_size is sentinel:
client_max_size = self._client_max_size
return self.__class__(
message,
self._payload,
self._protocol,
self._payload_writer,
self._task,
self._loop,
client_max_size=client_max_size,
state=self._state.copy(),
**kwargs,
)
@property
def task(self) -> "asyncio.Task[None]":
return self._task
@property
def protocol(self) -> "RequestHandler":
return self._protocol
@property
def transport(self) -> asyncio.Transport | None:
if self._protocol is None:
return None
return self._protocol.transport
@property
def writer(self) -> AbstractStreamWriter:
return self._payload_writer
@property
def client_max_size(self) -> int:
return self._client_max_size
@reify
def message(self) -> RawRequestMessage:
warnings.warn("Request.message is deprecated", DeprecationWarning, stacklevel=3)
return self._message
@reify
def rel_url(self) -> URL:
return self._rel_url
@reify
def loop(self) -> asyncio.AbstractEventLoop:
warnings.warn(
"request.loop property is deprecated", DeprecationWarning, stacklevel=2
)
return self._loop
# MutableMapping API
@overload # type: ignore[override]
def __getitem__(self, key: RequestKey[_T]) -> _T: ...
@overload
def __getitem__(self, key: str) -> Any: ...
def __getitem__(self, key: str | RequestKey[_T]) -> Any:
return self._state[key]
@overload # type: ignore[override]
def __setitem__(self, key: RequestKey[_T], value: _T) -> None: ...
@overload
def __setitem__(self, key: str, value: Any) -> None: ...
def __setitem__(self, key: str | RequestKey[_T], value: Any) -> None:
if not isinstance(key, RequestKey) and key not in BaseRequest._seen_str_keys:
BaseRequest._seen_str_keys.add(key)
warnings.warn(
"It is recommended to use web.RequestKey instances for keys.\n"
+ "https://docs.aiohttp.org/en/stable/web_advanced.html"
+ "#request-s-storage",
category=NotAppKeyWarning,
stacklevel=2,
)
self._state[key] = value
def __delitem__(self, key: str | RequestKey[_T]) -> None:
del self._state[key]
def __len__(self) -> int:
return len(self._state)
def __iter__(self) -> Iterator[str | RequestKey[Any]]:
return iter(self._state)
########
@reify
def secure(self) -> bool:
"""A bool indicating if the request is handled with SSL."""
return self.scheme == "https"
@reify
def forwarded(self) -> tuple[Mapping[str, str], ...]:
"""A tuple containing all parsed Forwarded header(s).
Makes an effort to parse Forwarded headers as specified by RFC 7239:
- It adds one (immutable) dictionary per Forwarded 'field-value', ie
per proxy. The element corresponds to the data in the Forwarded
field-value added by the first proxy encountered by the client. Each
subsequent item corresponds to those added by later proxies.
- It checks that every value has valid syntax in general as specified
in section 4: either a 'token' or a 'quoted-string'.
- It un-escapes found escape sequences.
- It does NOT validate 'by' and 'for' contents as specified in section
6.
- It does NOT validate 'host' contents (Host ABNF).
- It does NOT validate 'proto' contents for valid URI scheme names.
Returns a tuple containing one or more immutable dicts
"""
elems = []
for field_value in self._message.headers.getall(hdrs.FORWARDED, ()):
length = len(field_value)
pos = 0
need_separator = False
elem: dict[str, str] = {}
elems.append(types.MappingProxyType(elem))
while 0 <= pos < length:
match = _FORWARDED_PAIR_RE.match(field_value, pos)
if match is not None: # got a valid forwarded-pair
if need_separator:
# bad syntax here, skip to next comma
pos = field_value.find(",", pos)
else:
name, value, port = match.groups()
if value[0] == '"':
# quoted string: remove quotes and unescape
value = _QUOTED_PAIR_REPLACE_RE.sub(r"\1", value[1:-1])
if port:
value += port
elem[name.lower()] = value
pos += len(match.group(0))
need_separator = True
elif field_value[pos] == ",": # next forwarded-element
need_separator = False
elem = {}
elems.append(types.MappingProxyType(elem))
pos += 1
elif field_value[pos] == ";": # next forwarded-pair
need_separator = False
pos += 1
elif field_value[pos] in " \t":
# Allow whitespace even between forwarded-pairs, though
# RFC 7239 doesn't. This simplifies code and is in line
# with Postel's law.
pos += 1
else:
# bad syntax here, skip to next comma
pos = field_value.find(",", pos)
return tuple(elems)
@reify
def scheme(self) -> str:
"""A string representing the scheme of the request.
Hostname is resolved in this order:
- overridden value by .clone(scheme=new_scheme) call.
- type of connection to peer: HTTPS if socket is SSL, HTTP otherwise.
'http' or 'https'.
"""
if self._transport_sslcontext:
return "https"
else:
return "http"
@reify
def method(self) -> str:
"""Read only property for getting HTTP method.
The value is upper-cased str like 'GET', 'POST', 'PUT' etc.
"""
return self._method
@reify
def version(self) -> HttpVersion:
"""Read only property for getting HTTP version of request.
Returns aiohttp.protocol.HttpVersion instance.
"""
return self._version
@reify
def host(self) -> str:
"""Hostname of the request.
Hostname is resolved in this order:
- overridden value by .clone(host=new_host) call.
- HOST HTTP header
- local socket address the request arrived on
(transport ``sockname``)
- empty string if no transport information is available
For example, 'example.com' or 'localhost:8080'.
For historical reasons, the port number may be included.
"""
host = self._message.headers.get(hdrs.HOST)
if host is not None:
return host
sockname = self._transport_sockname
if sockname is None:
return ""
if isinstance(sockname, tuple):
# AF_INET6 returns a 4-tuple (host, port, flowinfo, scopeid);
# bracket the bare address so it matches the Host-header shape
# and is a valid URL authority component.
if len(sockname) == 4:
return f"[{sockname[0]}]"
return str(sockname[0])
return str(sockname)
@reify
def remote(self) -> str | None:
"""Remote IP of client initiated HTTP request.
The IP is resolved in this order:
- overridden value by .clone(remote=new_remote) call.
- peername of opened socket
"""
if self._transport_peername is None:
return None
if isinstance(self._transport_peername, (list, tuple)):
return str(self._transport_peername[0])
return str(self._transport_peername)
@reify
def url(self) -> URL:
"""The full URL of the request."""
# authority is used here because it may include the port number
# and we want yarl to parse it correctly
return URL.build(scheme=self.scheme, authority=self.host).join(self._rel_url)
@reify
def path(self) -> str:
"""The URL including *PATH INFO* without the host or scheme.
E.g., ``/app/blog``
"""
return self._rel_url.path
@reify
def path_qs(self) -> str:
"""The URL including PATH_INFO and the query string.
E.g, /app/blog?id=10
"""
return str(self._rel_url)
@reify
def raw_path(self) -> str:
"""The URL including raw *PATH INFO* without the host or scheme.
Warning, the path is unquoted and may contains non valid URL characters
E.g., ``/my%2Fpath%7Cwith%21some%25strange%24characters``
"""
return self._message.path
@reify
def query(self) -> "MultiMapping[str]":
"""A multidict with all the variables in the query string."""
return self._rel_url.query
@reify
def query_string(self) -> str:
"""The query string in the URL.
E.g., id=10
"""
return self._rel_url.query_string
@reify
def headers(self) -> CIMultiDictProxy[str]:
"""A case-insensitive multidict proxy with all headers."""
return self._headers
@reify
def raw_headers(self) -> RawHeaders:
"""A sequence of pairs for all headers."""
return self._message.raw_headers
@reify
def if_modified_since(self) -> datetime.datetime | None:
"""The value of If-Modified-Since HTTP header, or None.
This header is represented as a `datetime` object.
"""
return parse_http_date(self.headers.get(hdrs.IF_MODIFIED_SINCE))
@reify
def if_unmodified_since(self) -> datetime.datetime | None:
"""The value of If-Unmodified-Since HTTP header, or None.
This header is represented as a `datetime` object.
"""
return parse_http_date(self.headers.get(hdrs.IF_UNMODIFIED_SINCE))
@staticmethod
def _etag_values(etag_header: str) -> Iterator[ETag]:
"""Extract `ETag` objects from raw header."""
if etag_header == ETAG_ANY:
yield ETag(
is_weak=False,
value=ETAG_ANY,
)
else:
for match in LIST_QUOTED_ETAG_RE.finditer(etag_header):
is_weak, value, garbage = match.group(2, 3, 4)
# Any symbol captured by 4th group means
# that the following sequence is invalid.
if garbage:
break
yield ETag(
is_weak=bool(is_weak),
value=value,
)
@classmethod
def _if_match_or_none_impl(
cls, header_value: str | None
) -> tuple[ETag, ...] | None:
if not header_value:
return None
return tuple(cls._etag_values(header_value))
@reify
def if_match(self) -> tuple[ETag, ...] | None:
"""The value of If-Match HTTP header, or None.
This header is represented as a `tuple` of `ETag` objects.
"""
return self._if_match_or_none_impl(self.headers.get(hdrs.IF_MATCH))
@reify
def if_none_match(self) -> tuple[ETag, ...] | None:
"""The value of If-None-Match HTTP header, or None.
This header is represented as a `tuple` of `ETag` objects.
"""
return self._if_match_or_none_impl(self.headers.get(hdrs.IF_NONE_MATCH))
@reify
def if_range(self) -> datetime.datetime | None:
"""The value of If-Range HTTP header, or None.
This header is represented as a `datetime` object.
"""
return parse_http_date(self.headers.get(hdrs.IF_RANGE))
@reify
def keep_alive(self) -> bool:
"""Is keepalive enabled by client?"""
return not self._message.should_close
@reify
def cookies(self) -> Mapping[str, str]:
"""Return request cookies.
A read-only dictionary-like object.
"""
# Use parse_cookie_header for RFC 6265 compliant Cookie header parsing
# that accepts special characters in cookie names (fixes #2683)
parsed = parse_cookie_header(self.headers.get(hdrs.COOKIE, ""))
# Extract values from Morsel objects
return MappingProxyType({name: morsel.value for name, morsel in parsed})
@reify
def http_range(self) -> slice:
"""The content of Range HTTP header.
Return a slice instance.
"""
rng = self._headers.get(hdrs.RANGE)
start, end = None, None
if rng is not None:
try:
pattern = r"^bytes=(\d*)-(\d*)$"
start, end = re.findall(pattern, rng, re.ASCII)[0]
except IndexError: # pattern was not found in header
raise ValueError("range not in acceptable format")
end = int(end) if end else None
start = int(start) if start else None
if start is None and end is not None:
# end with no start is to return tail of content
start = -end
end = None
if start is not None and end is not None:
# end is inclusive in range header, exclusive for slice
end += 1
if start >= end:
raise ValueError("start cannot be after end")
if start is end is None: # No valid range supplied
raise ValueError("No start or end of range specified")
return slice(start, end, 1)
@reify
def content(self) -> StreamReader:
"""Return raw payload stream."""
return self._payload
@property
def has_body(self) -> bool:
"""Return True if request's HTTP BODY can be read, False otherwise."""
warnings.warn(
"Deprecated, use .can_read_body #2005", DeprecationWarning, stacklevel=2
)
return not self._payload.at_eof()
@property
def can_read_body(self) -> bool:
"""Return True if request's HTTP BODY can be read, False otherwise."""
return not self._payload.at_eof()
@reify
def body_exists(self) -> bool:
"""Return True if request has HTTP BODY, False otherwise."""
return type(self._payload) is not EmptyStreamReader
async def release(self) -> None:
"""Release request.
Eat unread part of HTTP BODY if present.
"""
while not self._payload.at_eof():
await self._payload.readany()
async def read(self) -> bytes:
"""Read request body if present.
Returns bytes object with full request content.
"""
if self._read_bytes is None:
# Raise the buffer limits so compressed payloads decompress in
# larger chunks instead of many small pause/resume cycles.
if self._client_max_size:
self._payload.set_read_chunk_size(self._client_max_size)
body = bytearray()
while True:
chunk = await self._payload.readany()
body.extend(chunk)
if self._client_max_size:
body_size = len(body)
if body_size > self._client_max_size:
raise HTTPRequestEntityTooLarge(self._client_max_size)
if not chunk:
break
self._read_bytes = bytes(body)
return self._read_bytes
async def text(self) -> str:
"""Return BODY as text using encoding from .charset."""
bytes_body = await self.read()
encoding = self.charset or "utf-8"
return bytes_body.decode(encoding)
async def json(self, *, loads: JSONDecoder = DEFAULT_JSON_DECODER) -> Any:
"""Return BODY as JSON."""
body = await self.text()
return loads(body)
async def multipart(self) -> MultipartReader:
"""Return async iterator to process BODY as multipart."""
return MultipartReader(
self._headers,
self._payload,
client_max_size=self._client_max_size,
max_field_size=self._protocol.max_field_size,
max_headers=self._protocol.max_headers,
max_size_error_cls=HTTPRequestEntityTooLarge,
)
async def post(self) -> "MultiDictProxy[_Post]":
"""Return POST parameters."""
if self._post is not None:
return self._post
if self._method not in self.POST_METHODS:
self._post = MultiDictProxy(MultiDict())
return self._post
content_type = self.content_type
if content_type not in (
"",
"application/x-www-form-urlencoded",
"multipart/form-data",
):
self._post = MultiDictProxy(MultiDict())
return self._post
out: MultiDict[_Post] = MultiDict()
if content_type == "multipart/form-data":
multipart = await self.multipart()
max_size = self._client_max_size
size = 0
while (field := await multipart.next()) is not None:
field_ct = field.headers.get(hdrs.CONTENT_TYPE)
if isinstance(field, BodyPartReader):
if field.name is None:
raise ValueError("Multipart field missing name.")
# Note that according to RFC 7578, the Content-Type header
# is optional, even for files, so we can't assume it's
# present.
# https://tools.ietf.org/html/rfc7578#section-4.4
if field.filename:
# store file in temp file
tmp = await self._loop.run_in_executor(
None, tempfile.TemporaryFile
)
while chunk := await field.read_chunk(size=DEFAULT_CHUNK_SIZE):
async for decoded_chunk in field.decode_iter(chunk):
await self._loop.run_in_executor(
None, tmp.write, decoded_chunk
)
size += len(decoded_chunk)
if 0 < max_size < size:
await self._loop.run_in_executor(None, tmp.close)
raise HTTPRequestEntityTooLarge(max_size)
await self._loop.run_in_executor(None, tmp.seek, 0)
if field_ct is None:
field_ct = "application/octet-stream"
ff = FileField(
field.name,
field.filename,
cast(io.BufferedReader, tmp),
field_ct,
field.headers,
)
out.add(field.name, ff)
else:
# deal with ordinary data
raw_data = bytearray()
while chunk := await field.read_chunk():
size += len(chunk)
if 0 < max_size < size:
raise HTTPRequestEntityTooLarge(max_size)
raw_data.extend(chunk)
value = bytearray()
# form-data doesn't support compression, so don't need to check size again.
async for d in field.decode_iter(raw_data):
value.extend(d)
if field_ct is None or field_ct.startswith("text/"):
charset = field.get_charset(default="utf-8")
out.add(field.name, value.decode(charset))
else:
out.add(field.name, value)
else:
raise ValueError(
"To decode nested multipart you need to use custom reader",
)
else:
data = await self.read()
if data:
charset = self.charset or "utf-8"
out.extend(
parse_qsl(
data.rstrip().decode(charset),
keep_blank_values=True,
encoding=charset,
)
)
self._post = MultiDictProxy(out)
return self._post
def get_extra_info(self, name: str, default: Any = None) -> Any:
"""Extra info from protocol transport"""
protocol = self._protocol
if protocol is None:
return default
transport = protocol.transport
if transport is None:
return default
return transport.get_extra_info(name, default)
def __repr__(self) -> str:
ascii_encodable_path = self.path.encode("ascii", "backslashreplace").decode(
"ascii"
)
return f"<{self.__class__.__name__} {self._method} {ascii_encodable_path} >"
def __eq__(self, other: object) -> bool:
return id(self) == id(other)
def __bool__(self) -> bool:
return True
async def _prepare_hook(self, response: StreamResponse) -> None:
return
def _cancel(self, exc: BaseException) -> None:
set_exception(self._payload, exc)
def _finish(self) -> None:
if self._post is None or self.content_type != "multipart/form-data":
return
# NOTE: Release file descriptors for the
# NOTE: `tempfile.Temporaryfile`-created `_io.BufferedRandom`
# NOTE: instances of files sent within multipart request body
# NOTE: via HTTP POST request.
for file_name, file_field_object in self._post.items():
if isinstance(file_field_object, FileField):
file_field_object.file.close()
class Request(BaseRequest):
ATTRS = BaseRequest.ATTRS | frozenset(["_match_info"])
_match_info: Optional["UrlMappingMatchInfo"] = None
if DEBUG:
def __setattr__(self, name: str, val: Any) -> None:
if name not in self.ATTRS:
warnings.warn(
f"Setting custom {self.__class__.__name__}.{name} attribute "
"is discouraged",
DeprecationWarning,
stacklevel=2,
)
super().__setattr__(name, val)
def clone(
self,
*,
method: str | _SENTINEL = sentinel,
rel_url: StrOrURL | _SENTINEL = sentinel,
headers: LooseHeaders | _SENTINEL = sentinel,
scheme: str | _SENTINEL = sentinel,
host: str | _SENTINEL = sentinel,
remote: str | _SENTINEL = sentinel,
client_max_size: int | _SENTINEL = sentinel,
) -> "Request":
ret = super().clone(
method=method,
rel_url=rel_url,
headers=headers,
scheme=scheme,
host=host,
remote=remote,
client_max_size=client_max_size,
)
new_ret = cast(Request, ret)
new_ret._match_info = self._match_info
return new_ret
@reify
def match_info(self) -> "UrlMappingMatchInfo":
"""Result of route resolving."""
match_info = self._match_info
assert match_info is not None
return match_info
@property
def app(self) -> "Application":
"""Application instance."""
match_info = self._match_info
assert match_info is not None
return match_info.current_app
@property
def config_dict(self) -> ChainMapProxy:
match_info = self._match_info
assert match_info is not None
lst = match_info.apps
app = self.app
idx = lst.index(app)
sublist = list(reversed(lst[: idx + 1]))
return ChainMapProxy(sublist)
async def _prepare_hook(self, response: StreamResponse) -> None:
match_info = self._match_info
if match_info is None:
return
for app in match_info._apps:
if on_response_prepare := app.on_response_prepare:
await on_response_prepare.send(self, response)
+911
View File
@@ -0,0 +1,911 @@
import asyncio
import collections.abc
import datetime
import enum
import json
import math
import time
import warnings
from collections.abc import Iterator, MutableMapping
from concurrent.futures import Executor
from http import HTTPStatus
from http.cookies import SimpleCookie
from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast, overload
from multidict import CIMultiDict, istr
from . import hdrs, payload
from .abc import AbstractStreamWriter
from .compression_utils import MAX_SYNC_CHUNK_SIZE, ZLibCompressor
from .helpers import (
ETAG_ANY,
QUOTED_ETAG_RE,
ETag,
HeadersMixin,
ResponseKey,
must_be_empty_body,
parse_http_date,
rfc822_formatted_time,
sentinel,
should_remove_content_length,
validate_etag_value,
)
from .http import SERVER_SOFTWARE, HttpVersion10, HttpVersion11
from .payload import Payload
from .typedefs import JSONBytesEncoder, JSONEncoder, LooseHeaders
REASON_PHRASES = {http_status.value: http_status.phrase for http_status in HTTPStatus}
__all__ = (
"ContentCoding",
"StreamResponse",
"Response",
"json_response",
"json_bytes_response",
)
if TYPE_CHECKING:
from .web_request import BaseRequest
BaseClass = MutableMapping[str, Any]
else:
BaseClass = collections.abc.MutableMapping
_T = TypeVar("_T")
# TODO(py311): Convert to StrEnum for wider use
class ContentCoding(enum.Enum):
# The content codings that we have support for.
#
# Additional registered codings are listed at:
# https://www.iana.org/assignments/http-parameters/http-parameters.xhtml#content-coding
deflate = "deflate"
gzip = "gzip"
identity = "identity"
CONTENT_CODINGS = {coding.value: coding for coding in ContentCoding}
############################################################
# HTTP Response classes
############################################################
class StreamResponse(MutableMapping[str | ResponseKey[Any], Any], HeadersMixin):
_body: None | bytes | bytearray | Payload
_length_check = True
_body = None
_keep_alive: bool | None = None
_chunked: bool = False
_compression: bool = False
_compression_strategy: int | None = None
_compression_force: ContentCoding | None = None
_req: Optional["BaseRequest"] = None
_payload_writer: AbstractStreamWriter | None = None
_eof_sent: bool = False
_must_be_empty_body: bool | None = None
_body_length = 0
_cookies: SimpleCookie | None = None
_send_headers_immediately = True
_seen_str_keys: set[str] = set()
def __init__(
self,
*,
status: int = 200,
reason: str | None = None,
headers: LooseHeaders | None = None,
_real_headers: CIMultiDict[str] | None = None,
) -> None:
"""Initialize a new stream response object.
_real_headers is an internal parameter used to pass a pre-populated
headers object. It is used by the `Response` class to avoid copying
the headers when creating a new response object. It is not intended
to be used by external code.
"""
self._state: dict[str | ResponseKey[Any], Any] = {}
if _real_headers is not None:
self._headers = _real_headers
elif headers is not None:
self._headers: CIMultiDict[str] = CIMultiDict(headers)
else:
self._headers = CIMultiDict()
self._set_status(status, reason)
@property
def prepared(self) -> bool:
return self._eof_sent or self._payload_writer is not None
@property
def task(self) -> "asyncio.Task[None] | None":
if self._req:
return self._req.task
else:
return None
@property
def status(self) -> int:
return self._status
@property
def chunked(self) -> bool:
return self._chunked
@property
def compression(self) -> bool:
return self._compression
@property
def reason(self) -> str:
return self._reason
def set_status(
self,
status: int,
reason: str | None = None,
) -> None:
assert (
not self.prepared
), "Cannot change the response status code after the headers have been sent"
self._set_status(status, reason)
def _set_status(self, status: int, reason: str | None) -> None:
self._status = int(status)
if reason is None:
reason = REASON_PHRASES.get(self._status, "")
elif "\r" in reason or "\n" in reason:
raise ValueError("Reason cannot contain \\r or \\n")
self._reason = reason
@property
def keep_alive(self) -> bool | None:
return self._keep_alive
def force_close(self) -> None:
self._keep_alive = False
@property
def body_length(self) -> int:
return self._body_length
@property
def output_length(self) -> int:
warnings.warn("output_length is deprecated", DeprecationWarning)
assert self._payload_writer
return self._payload_writer.buffer_size
def enable_chunked_encoding(self, chunk_size: int | None = None) -> None:
"""Enables automatic chunked transfer encoding."""
if hdrs.CONTENT_LENGTH in self._headers:
raise RuntimeError(
"You can't enable chunked encoding when a content length is set"
)
if chunk_size is not None:
warnings.warn("Chunk size is deprecated #1615", DeprecationWarning)
self._chunked = True
def enable_compression(
self,
force: bool | ContentCoding | None = None,
strategy: int | None = None,
) -> None:
"""Enables response compression encoding."""
# Backwards compatibility for when force was a bool <0.17.
if isinstance(force, bool):
force = ContentCoding.deflate if force else ContentCoding.identity
warnings.warn(
"Using boolean for force is deprecated #3318", DeprecationWarning
)
elif force is not None:
assert isinstance(
force, ContentCoding
), "force should one of None, bool or ContentEncoding"
self._compression = True
self._compression_force = force
self._compression_strategy = strategy
@property
def headers(self) -> "CIMultiDict[str]":
return self._headers
@property
def cookies(self) -> SimpleCookie:
if self._cookies is None:
self._cookies = SimpleCookie()
return self._cookies
def set_cookie(
self,
name: str,
value: str,
*,
expires: str | None = None,
domain: str | None = None,
max_age: int | str | None = None,
path: str = "/",
secure: bool | None = None,
httponly: bool | None = None,
version: str | None = None,
samesite: str | None = None,
partitioned: bool | None = None,
) -> None:
"""Set or update response cookie.
Sets new cookie or updates existent with new value.
Also updates only those params which are not None.
"""
if self._cookies is None:
self._cookies = SimpleCookie()
self._cookies[name] = value
c = self._cookies[name]
if expires is not None:
c["expires"] = expires
elif c.get("expires") == "Thu, 01 Jan 1970 00:00:00 GMT":
del c["expires"]
if domain is not None:
c["domain"] = domain
if max_age is not None:
c["max-age"] = str(max_age)
elif "max-age" in c:
del c["max-age"]
c["path"] = path
if secure is not None:
c["secure"] = secure
if httponly is not None:
c["httponly"] = httponly
if version is not None:
c["version"] = version
if samesite is not None:
c["samesite"] = samesite
if partitioned is not None:
c["partitioned"] = partitioned
def del_cookie(
self,
name: str,
*,
domain: str | None = None,
path: str = "/",
secure: bool | None = None,
httponly: bool | None = None,
samesite: str | None = None,
) -> None:
"""Delete cookie.
Creates new empty expired cookie.
"""
# TODO: do we need domain/path here?
if self._cookies is not None:
self._cookies.pop(name, None)
self.set_cookie(
name,
"",
max_age=0,
expires="Thu, 01 Jan 1970 00:00:00 GMT",
domain=domain,
path=path,
secure=secure,
httponly=httponly,
samesite=samesite,
)
@property
def content_length(self) -> int | None:
# Just a placeholder for adding setter
return super().content_length
@content_length.setter
def content_length(self, value: int | None) -> None:
if value is not None:
value = int(value)
if self._chunked:
raise RuntimeError(
"You can't set content length when chunked encoding is enable"
)
self._headers[hdrs.CONTENT_LENGTH] = str(value)
else:
self._headers.pop(hdrs.CONTENT_LENGTH, None)
@property
def content_type(self) -> str:
# Just a placeholder for adding setter
return super().content_type
@content_type.setter
def content_type(self, value: str) -> None:
self.content_type # read header values if needed
self._content_type = str(value)
self._generate_content_type_header()
@property
def charset(self) -> str | None:
# Just a placeholder for adding setter
return super().charset
@charset.setter
def charset(self, value: str | None) -> None:
ctype = self.content_type # read header values if needed
if ctype == "application/octet-stream":
raise RuntimeError(
"Setting charset for application/octet-stream "
"doesn't make sense, setup content_type first"
)
assert self._content_dict is not None
if value is None:
self._content_dict.pop("charset", None)
else:
self._content_dict["charset"] = str(value).lower()
self._generate_content_type_header()
@property
def last_modified(self) -> datetime.datetime | None:
"""The value of Last-Modified HTTP header, or None.
This header is represented as a `datetime` object.
"""
return parse_http_date(self._headers.get(hdrs.LAST_MODIFIED))
@last_modified.setter
def last_modified(
self, value: int | float | datetime.datetime | str | None
) -> None:
if value is None:
self._headers.pop(hdrs.LAST_MODIFIED, None)
elif isinstance(value, (int, float)):
self._headers[hdrs.LAST_MODIFIED] = time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", time.gmtime(math.ceil(value))
)
elif isinstance(value, datetime.datetime):
if value.microsecond:
value = value.replace(microsecond=0) + datetime.timedelta(seconds=1)
self._headers[hdrs.LAST_MODIFIED] = time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", value.utctimetuple()
)
elif isinstance(value, str):
self._headers[hdrs.LAST_MODIFIED] = value
else:
msg = f"Unsupported type for last_modified: {type(value).__name__}"
raise TypeError(msg)
@property
def etag(self) -> ETag | None:
quoted_value = self._headers.get(hdrs.ETAG)
if not quoted_value:
return None
elif quoted_value == ETAG_ANY:
return ETag(value=ETAG_ANY)
match = QUOTED_ETAG_RE.fullmatch(quoted_value)
if not match:
return None
is_weak, value = match.group(1, 2)
return ETag(
is_weak=bool(is_weak),
value=value,
)
@etag.setter
def etag(self, value: ETag | str | None) -> None:
if value is None:
self._headers.pop(hdrs.ETAG, None)
elif (isinstance(value, str) and value == ETAG_ANY) or (
isinstance(value, ETag) and value.value == ETAG_ANY
):
self._headers[hdrs.ETAG] = ETAG_ANY
elif isinstance(value, str):
validate_etag_value(value)
self._headers[hdrs.ETAG] = f'"{value}"'
elif isinstance(value, ETag) and isinstance(value.value, str):
validate_etag_value(value.value)
hdr_value = f'W/"{value.value}"' if value.is_weak else f'"{value.value}"'
self._headers[hdrs.ETAG] = hdr_value
else:
raise ValueError(
f"Unsupported etag type: {type(value)}. "
f"etag must be str, ETag or None"
)
def _generate_content_type_header(
self, CONTENT_TYPE: istr = hdrs.CONTENT_TYPE
) -> None:
assert self._content_dict is not None
assert self._content_type is not None
params = "; ".join(f"{k}={v}" for k, v in self._content_dict.items())
if params:
ctype = self._content_type + "; " + params
else:
ctype = self._content_type
self._headers[CONTENT_TYPE] = ctype
async def _do_start_compression(self, coding: ContentCoding) -> None:
if coding is ContentCoding.identity:
return
assert self._payload_writer is not None
self._headers[hdrs.CONTENT_ENCODING] = coding.value
self._payload_writer.enable_compression(
coding.value, self._compression_strategy
)
# Compressed payload may have different content length,
# remove the header
self._headers.popall(hdrs.CONTENT_LENGTH, None)
async def _start_compression(self, request: "BaseRequest") -> None:
if self._compression_force:
await self._do_start_compression(self._compression_force)
return
# Encoding comparisons should be case-insensitive
# https://www.rfc-editor.org/rfc/rfc9110#section-8.4.1
accept_encoding = request.headers.get(hdrs.ACCEPT_ENCODING, "").lower()
for value, coding in CONTENT_CODINGS.items():
if value in accept_encoding:
await self._do_start_compression(coding)
return
async def prepare(self, request: "BaseRequest") -> AbstractStreamWriter | None:
if self._eof_sent:
return None
if self._payload_writer is not None:
return self._payload_writer
self._must_be_empty_body = must_be_empty_body(request.method, self.status)
return await self._start(request)
async def _start(self, request: "BaseRequest") -> AbstractStreamWriter:
self._req = request
writer = self._payload_writer = request._payload_writer
await self._prepare_headers()
await request._prepare_hook(self)
await self._write_headers()
return writer
async def _prepare_headers(self) -> None:
request = self._req
assert request is not None
writer = self._payload_writer
assert writer is not None
keep_alive = self._keep_alive
if keep_alive is None:
keep_alive = request.keep_alive
self._keep_alive = keep_alive
version = request.version
headers = self._headers
if self._cookies:
for cookie in self._cookies.values():
value = cookie.output(header="")[1:]
headers.add(hdrs.SET_COOKIE, value)
if self._compression:
await self._start_compression(request)
if self._chunked:
if version != HttpVersion11:
raise RuntimeError(
"Using chunked encoding is forbidden "
f"for HTTP/{request.version.major}.{request.version.minor}"
)
if not self._must_be_empty_body:
writer.enable_chunking()
headers[hdrs.TRANSFER_ENCODING] = "chunked"
elif self._length_check: # Disabled for WebSockets
writer.length = self.content_length
if writer.length is None:
if version >= HttpVersion11:
if not self._must_be_empty_body:
writer.enable_chunking()
headers[hdrs.TRANSFER_ENCODING] = "chunked"
elif not self._must_be_empty_body:
keep_alive = False
# HTTP 1.1: https://tools.ietf.org/html/rfc7230#section-3.3.2
# HTTP 1.0: https://tools.ietf.org/html/rfc1945#section-10.4
if self._must_be_empty_body:
if hdrs.CONTENT_LENGTH in headers and should_remove_content_length(
request.method, self.status
):
del headers[hdrs.CONTENT_LENGTH]
# https://datatracker.ietf.org/doc/html/rfc9112#section-6.1-10
# https://datatracker.ietf.org/doc/html/rfc9112#section-6.1-13
if hdrs.TRANSFER_ENCODING in headers:
del headers[hdrs.TRANSFER_ENCODING]
elif (writer.length if self._length_check else self.content_length) != 0:
# https://www.rfc-editor.org/rfc/rfc9110#section-8.3-5
headers.setdefault(hdrs.CONTENT_TYPE, "application/octet-stream")
headers.setdefault(hdrs.DATE, rfc822_formatted_time())
headers.setdefault(hdrs.SERVER, SERVER_SOFTWARE)
# connection header
if hdrs.CONNECTION not in headers:
if keep_alive:
if version == HttpVersion10:
headers[hdrs.CONNECTION] = "keep-alive"
elif version == HttpVersion11:
headers[hdrs.CONNECTION] = "close"
async def _write_headers(self) -> None:
request = self._req
assert request is not None
writer = self._payload_writer
assert writer is not None
# status line
version = request.version
status_line = f"HTTP/{version[0]}.{version[1]} {self._status} {self._reason}"
await writer.write_headers(status_line, self._headers)
# Send headers immediately if not opted into buffering
if self._send_headers_immediately:
writer.send_headers()
async def write(self, data: bytes | bytearray | memoryview) -> None:
assert isinstance(
data, (bytes, bytearray, memoryview)
), "data argument must be byte-ish (%r)" % type(data)
if self._eof_sent:
raise RuntimeError("Cannot call write() after write_eof()")
if self._payload_writer is None:
raise RuntimeError("Cannot call write() before prepare()")
await self._payload_writer.write(data)
async def drain(self) -> None:
assert not self._eof_sent, "EOF has already been sent"
assert self._payload_writer is not None, "Response has not been started"
warnings.warn(
"drain method is deprecated, use await resp.write()",
DeprecationWarning,
stacklevel=2,
)
await self._payload_writer.drain()
async def write_eof(self, data: bytes = b"") -> None:
assert isinstance(
data, (bytes, bytearray, memoryview)
), "data argument must be byte-ish (%r)" % type(data)
if self._eof_sent:
return
assert self._payload_writer is not None, "Response has not been started"
await self._payload_writer.write_eof(data)
self._eof_sent = True
self._req = None
self._body_length = self._payload_writer.output_size
self._payload_writer = None
def __repr__(self) -> str:
if self._eof_sent:
info = "eof"
elif self.prepared:
assert self._req is not None
info = f"{self._req.method} {self._req.path} "
else:
info = "not prepared"
return f"<{self.__class__.__name__} {self.reason} {info}>"
@overload # type: ignore[override]
def __getitem__(self, key: ResponseKey[_T]) -> _T: ...
@overload
def __getitem__(self, key: str) -> Any: ...
def __getitem__(self, key: str | ResponseKey[_T]) -> Any:
return self._state[key]
@overload # type: ignore[override]
def __setitem__(self, key: ResponseKey[_T], value: _T) -> None: ...
@overload
def __setitem__(self, key: str, value: Any) -> None: ...
def __setitem__(self, key: str | ResponseKey[_T], value: Any) -> None:
if (
not isinstance(key, ResponseKey)
and key not in StreamResponse._seen_str_keys
):
# Import here to break circular dependency
from .web_exceptions import NotAppKeyWarning
StreamResponse._seen_str_keys.add(key)
warnings.warn(
"It is recommended to use web.ResponseKey instances for keys.\n"
+ "https://docs.aiohttp.org/en/stable/web_advanced.html"
+ "#response-s-storage",
category=NotAppKeyWarning,
stacklevel=2,
)
self._state[key] = value
def __delitem__(self, key: str | ResponseKey[_T]) -> None:
del self._state[key]
def __len__(self) -> int:
return len(self._state)
def __iter__(self) -> Iterator[str | ResponseKey[Any]]:
return iter(self._state)
def __hash__(self) -> int:
return hash(id(self))
def __eq__(self, other: object) -> bool:
return self is other
def __bool__(self) -> bool:
return True
class Response(StreamResponse):
_compressed_body: bytes | None = None
_send_headers_immediately = False
def __init__(
self,
*,
body: Any = None,
status: int = 200,
reason: str | None = None,
text: str | None = None,
headers: LooseHeaders | None = None,
content_type: str | None = None,
charset: str | None = None,
zlib_executor_size: int = MAX_SYNC_CHUNK_SIZE,
zlib_executor: Executor | None = None,
) -> None:
if body is not None and text is not None:
raise ValueError("body and text are not allowed together")
if headers is None:
real_headers: CIMultiDict[str] = CIMultiDict()
else:
real_headers = CIMultiDict(headers)
if content_type is not None and "charset" in content_type:
raise ValueError("charset must not be in content_type argument")
if text is not None:
if hdrs.CONTENT_TYPE in real_headers:
if content_type or charset:
raise ValueError(
"passing both Content-Type header and "
"content_type or charset params "
"is forbidden"
)
else:
# fast path for filling headers
if not isinstance(text, str):
raise TypeError("text argument must be str (%r)" % type(text))
if content_type is None:
content_type = "text/plain"
if charset is None:
charset = "utf-8"
real_headers[hdrs.CONTENT_TYPE] = content_type + "; charset=" + charset
body = text.encode(charset)
text = None
elif hdrs.CONTENT_TYPE in real_headers:
if content_type is not None or charset is not None:
raise ValueError(
"passing both Content-Type header and "
"content_type or charset params "
"is forbidden"
)
elif content_type is not None:
if charset is not None:
content_type += "; charset=" + charset
real_headers[hdrs.CONTENT_TYPE] = content_type
super().__init__(status=status, reason=reason, _real_headers=real_headers)
if text is not None:
self.text = text
else:
self.body = body
self._zlib_executor_size = zlib_executor_size
self._zlib_executor = zlib_executor
@property
def body(self) -> bytes | bytearray | Payload | None:
return self._body
@body.setter
def body(self, body: Any) -> None:
if body is None:
self._body = None
elif isinstance(body, (bytes, bytearray)):
self._body = body
else:
try:
self._body = body = payload.PAYLOAD_REGISTRY.get(body)
except payload.LookupError:
raise ValueError("Unsupported body type %r" % type(body))
headers = self._headers
# set content-type
if hdrs.CONTENT_TYPE not in headers:
headers[hdrs.CONTENT_TYPE] = body.content_type
# copy payload headers
if body.headers:
for key, value in body.headers.items():
if key not in headers:
headers[key] = value
self._compressed_body = None
@property
def text(self) -> str | None:
if self._body is None:
return None
# Note: When _body is a Payload (e.g. FilePayload), this may do blocking I/O
# This is generally safe as most common payloads (BytesPayload, StringPayload)
# don't do blocking I/O, but be careful with file-based payloads
return self._body.decode(self.charset or "utf-8")
@text.setter
def text(self, text: str) -> None:
assert text is None or isinstance(
text, str
), "text argument must be str (%r)" % type(text)
if self.content_type == "application/octet-stream":
self.content_type = "text/plain"
if self.charset is None:
self.charset = "utf-8"
self._body = text.encode(self.charset)
self._compressed_body = None
@property
def content_length(self) -> int | None:
if self._chunked:
return None
if hdrs.CONTENT_LENGTH in self._headers:
return int(self._headers[hdrs.CONTENT_LENGTH])
if self._compressed_body is not None:
# Return length of the compressed body
return len(self._compressed_body)
elif isinstance(self._body, Payload):
# A payload without content length, or a compressed payload
return None
elif self._body is not None:
return len(self._body)
else:
return 0
@content_length.setter
def content_length(self, value: int | None) -> None:
raise RuntimeError("Content length is set automatically")
async def write_eof(self, data: bytes = b"") -> None:
if self._eof_sent:
return
if self._compressed_body is None:
body = self._body
else:
body = self._compressed_body
assert not data, f"data arg is not supported, got {data!r}"
assert self._req is not None
assert self._payload_writer is not None
if body is None or self._must_be_empty_body:
await super().write_eof()
elif isinstance(self._body, Payload):
try:
await self._body.write(self._payload_writer)
finally:
await self._body.close()
await super().write_eof()
else:
await super().write_eof(cast(bytes, body))
async def _start(self, request: "BaseRequest") -> AbstractStreamWriter:
if hdrs.CONTENT_LENGTH in self._headers:
if should_remove_content_length(request.method, self.status):
del self._headers[hdrs.CONTENT_LENGTH]
elif not self._chunked:
if isinstance(self._body, Payload):
if (size := self._body.size) is not None:
self._headers[hdrs.CONTENT_LENGTH] = str(size)
else:
body_len = len(self._body) if self._body else "0"
# https://www.rfc-editor.org/rfc/rfc9110.html#section-8.6-7
if body_len != "0" or (
self.status != 304 and request.method != hdrs.METH_HEAD
):
self._headers[hdrs.CONTENT_LENGTH] = str(body_len)
return await super()._start(request)
async def _do_start_compression(self, coding: ContentCoding) -> None:
if self._chunked or isinstance(self._body, Payload):
return await super()._do_start_compression(coding)
if coding is ContentCoding.identity:
return
# Instead of using _payload_writer.enable_compression,
# compress the whole body
compressor = ZLibCompressor(
encoding=coding.value,
max_sync_chunk_size=self._zlib_executor_size,
executor=self._zlib_executor,
)
assert self._body is not None
self._compressed_body = (
await compressor.compress(self._body) + compressor.flush()
)
self._headers[hdrs.CONTENT_ENCODING] = coding.value
self._headers[hdrs.CONTENT_LENGTH] = str(len(self._compressed_body))
def json_response(
data: Any = sentinel,
*,
text: str | None = None,
body: bytes | None = None,
status: int = 200,
reason: str | None = None,
headers: LooseHeaders | None = None,
content_type: str = "application/json",
dumps: JSONEncoder = json.dumps,
) -> Response:
if data is not sentinel:
if text or body:
raise ValueError("only one of data, text, or body should be specified")
else:
text = dumps(data)
return Response(
text=text,
body=body,
status=status,
reason=reason,
headers=headers,
content_type=content_type,
)
def json_bytes_response(
data: Any = sentinel,
*,
dumps: JSONBytesEncoder,
body: bytes | None = None,
status: int = 200,
reason: str | None = None,
headers: LooseHeaders | None = None,
content_type: str = "application/json",
) -> Response:
"""Create a JSON response using a bytes-returning encoder.
Use this when your JSON encoder (like orjson) returns bytes
instead of str, avoiding the encode/decode overhead.
"""
if data is not sentinel:
if body is not None:
raise ValueError("only one of data or body should be specified")
else:
body = dumps(data)
return Response(
body=body,
status=status,
reason=reason,
headers=headers,
content_type=content_type,
)
+203
View File
@@ -0,0 +1,203 @@
import abc
import os # noqa
from collections.abc import Callable, Iterator, Sequence
from typing import TYPE_CHECKING, Any, Union, overload
import attr
from . import hdrs
from .abc import AbstractView
from .typedefs import Handler, PathLike
if TYPE_CHECKING:
from .web_request import Request
from .web_response import StreamResponse
from .web_urldispatcher import AbstractRoute, UrlDispatcher
else:
Request = StreamResponse = UrlDispatcher = AbstractRoute = None
__all__ = (
"AbstractRouteDef",
"RouteDef",
"StaticDef",
"RouteTableDef",
"head",
"options",
"get",
"post",
"patch",
"put",
"delete",
"route",
"view",
"static",
)
class AbstractRouteDef(abc.ABC):
@abc.abstractmethod
def register(self, router: UrlDispatcher) -> list[AbstractRoute]:
pass # pragma: no cover
_HandlerType = Union[type[AbstractView], Handler]
@attr.s(auto_attribs=True, frozen=True, repr=False, slots=True)
class RouteDef(AbstractRouteDef):
method: str
path: str
handler: _HandlerType
kwargs: dict[str, Any]
def __repr__(self) -> str:
info = []
for name, value in sorted(self.kwargs.items()):
info.append(f", {name}={value!r}")
return "<RouteDef {method} {path} -> {handler.__name__!r}{info}>".format(
method=self.method, path=self.path, handler=self.handler, info="".join(info)
)
def register(self, router: UrlDispatcher) -> list[AbstractRoute]:
if self.method in hdrs.METH_ALL:
reg = getattr(router, "add_" + self.method.lower())
return [reg(self.path, self.handler, **self.kwargs)]
else:
return [
router.add_route(self.method, self.path, self.handler, **self.kwargs)
]
@attr.s(auto_attribs=True, frozen=True, repr=False, slots=True)
class StaticDef(AbstractRouteDef):
prefix: str
path: PathLike
kwargs: dict[str, Any]
def __repr__(self) -> str:
info = []
for name, value in sorted(self.kwargs.items()):
info.append(f", {name}={value!r}")
return "<StaticDef {prefix} -> {path}{info}>".format(
prefix=self.prefix, path=self.path, info="".join(info)
)
def register(self, router: UrlDispatcher) -> list[AbstractRoute]:
resource = router.add_static(self.prefix, self.path, **self.kwargs)
routes = resource.get_info().get("routes", {})
return list(routes.values())
def route(method: str, path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef:
return RouteDef(method, path, handler, kwargs)
def head(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef:
return route(hdrs.METH_HEAD, path, handler, **kwargs)
def options(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef:
return route(hdrs.METH_OPTIONS, path, handler, **kwargs)
def get(
path: str,
handler: _HandlerType,
*,
name: str | None = None,
allow_head: bool = True,
**kwargs: Any,
) -> RouteDef:
return route(
hdrs.METH_GET, path, handler, name=name, allow_head=allow_head, **kwargs
)
def post(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef:
return route(hdrs.METH_POST, path, handler, **kwargs)
def put(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef:
return route(hdrs.METH_PUT, path, handler, **kwargs)
def patch(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef:
return route(hdrs.METH_PATCH, path, handler, **kwargs)
def delete(path: str, handler: _HandlerType, **kwargs: Any) -> RouteDef:
return route(hdrs.METH_DELETE, path, handler, **kwargs)
def view(path: str, handler: type[AbstractView], **kwargs: Any) -> RouteDef:
return route(hdrs.METH_ANY, path, handler, **kwargs)
def static(prefix: str, path: PathLike, **kwargs: Any) -> StaticDef:
return StaticDef(prefix, path, kwargs)
_Deco = Callable[[_HandlerType], _HandlerType]
class RouteTableDef(Sequence[AbstractRouteDef]):
"""Route definition table"""
def __init__(self) -> None:
self._items: list[AbstractRouteDef] = []
def __repr__(self) -> str:
return f"<RouteTableDef count={len(self._items)}>"
@overload
def __getitem__(self, index: int) -> AbstractRouteDef: ...
@overload
def __getitem__(self, index: slice) -> list[AbstractRouteDef]: ...
def __getitem__(self, index): # type: ignore[no-untyped-def]
return self._items[index]
def __iter__(self) -> Iterator[AbstractRouteDef]:
return iter(self._items)
def __len__(self) -> int:
return len(self._items)
def __contains__(self, item: object) -> bool:
return item in self._items
def route(self, method: str, path: str, **kwargs: Any) -> _Deco:
def inner(handler: _HandlerType) -> _HandlerType:
self._items.append(RouteDef(method, path, handler, kwargs))
return handler
return inner
def head(self, path: str, **kwargs: Any) -> _Deco:
return self.route(hdrs.METH_HEAD, path, **kwargs)
def get(self, path: str, **kwargs: Any) -> _Deco:
return self.route(hdrs.METH_GET, path, **kwargs)
def post(self, path: str, **kwargs: Any) -> _Deco:
return self.route(hdrs.METH_POST, path, **kwargs)
def put(self, path: str, **kwargs: Any) -> _Deco:
return self.route(hdrs.METH_PUT, path, **kwargs)
def patch(self, path: str, **kwargs: Any) -> _Deco:
return self.route(hdrs.METH_PATCH, path, **kwargs)
def delete(self, path: str, **kwargs: Any) -> _Deco:
return self.route(hdrs.METH_DELETE, path, **kwargs)
def options(self, path: str, **kwargs: Any) -> _Deco:
return self.route(hdrs.METH_OPTIONS, path, **kwargs)
def view(self, path: str, **kwargs: Any) -> _Deco:
return self.route(hdrs.METH_ANY, path, **kwargs)
def static(self, prefix: str, path: PathLike, **kwargs: Any) -> None:
self._items.append(StaticDef(prefix, path, kwargs))
+425
View File
@@ -0,0 +1,425 @@
import asyncio
import signal
import socket
import warnings
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any
from yarl import URL
from .abc import AbstractAccessLogger
from .typedefs import PathLike
from .web_app import Application
from .web_log import AccessLogger
from .web_server import Server
if TYPE_CHECKING:
from ssl import SSLContext
else:
try:
from ssl import SSLContext
except ImportError: # pragma: no cover
SSLContext = object # type: ignore[misc,assignment]
__all__ = (
"BaseSite",
"TCPSite",
"UnixSite",
"NamedPipeSite",
"SockSite",
"BaseRunner",
"AppRunner",
"ServerRunner",
"GracefulExit",
)
class GracefulExit(SystemExit):
code = 1
def _raise_graceful_exit() -> None:
raise GracefulExit()
class BaseSite(ABC):
__slots__ = ("_runner", "_ssl_context", "_backlog", "_server")
def __init__(
self,
runner: "BaseRunner",
*,
shutdown_timeout: float = 60.0,
ssl_context: SSLContext | None = None,
backlog: int = 128,
) -> None:
if runner.server is None:
raise RuntimeError("Call runner.setup() before making a site")
if shutdown_timeout != 60.0:
msg = "shutdown_timeout should be set on BaseRunner"
warnings.warn(msg, DeprecationWarning, stacklevel=2)
runner._shutdown_timeout = shutdown_timeout
self._runner = runner
self._ssl_context = ssl_context
self._backlog = backlog
self._server: asyncio.AbstractServer | None = None
@property
@abstractmethod
def name(self) -> str:
pass # pragma: no cover
@abstractmethod
async def start(self) -> None:
self._runner._reg_site(self)
async def stop(self) -> None:
self._runner._check_site(self)
if self._server is not None: # Maybe not started yet
self._server.close()
self._runner._unreg_site(self)
class TCPSite(BaseSite):
__slots__ = ("_host", "_port", "_bound_port", "_reuse_address", "_reuse_port")
def __init__(
self,
runner: "BaseRunner",
host: str | None = None,
port: int | None = None,
*,
shutdown_timeout: float = 60.0,
ssl_context: SSLContext | None = None,
backlog: int = 128,
reuse_address: bool | None = None,
reuse_port: bool | None = None,
) -> None:
super().__init__(
runner,
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog,
)
self._host = host
if port is None:
port = 8443 if self._ssl_context else 8080
self._port = port
self._bound_port: int | None = None
self._reuse_address = reuse_address
self._reuse_port = reuse_port
@property
def port(self) -> int:
"""The port the server is listening on.
If the server hasn't been started yet, this returns the requested port
(which might be 0 for a dynamic port).
After the server starts, it returns the actual bound port. This is
especially useful when port=0 was requested, as it allows retrieving the
dynamically assigned port after the site has started.
"""
if self._bound_port is not None:
return self._bound_port
return self._port
@property
def name(self) -> str:
scheme = "https" if self._ssl_context else "http"
host = "0.0.0.0" if not self._host else self._host
return str(URL.build(scheme=scheme, host=host, port=self.port))
async def start(self) -> None:
await super().start()
loop = asyncio.get_event_loop()
server = self._runner.server
assert server is not None
self._server = await loop.create_server(
server,
self._host,
self._port,
ssl=self._ssl_context,
backlog=self._backlog,
reuse_address=self._reuse_address,
reuse_port=self._reuse_port,
)
if self._server.sockets:
self._bound_port = self._server.sockets[0].getsockname()[1]
else:
self._bound_port = self._port
class UnixSite(BaseSite):
__slots__ = ("_path",)
def __init__(
self,
runner: "BaseRunner",
path: PathLike,
*,
shutdown_timeout: float = 60.0,
ssl_context: SSLContext | None = None,
backlog: int = 128,
) -> None:
super().__init__(
runner,
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog,
)
self._path = path
@property
def name(self) -> str:
scheme = "https" if self._ssl_context else "http"
return f"{scheme}://unix:{self._path}:"
async def start(self) -> None:
await super().start()
loop = asyncio.get_event_loop()
server = self._runner.server
assert server is not None
self._server = await loop.create_unix_server(
server,
self._path,
ssl=self._ssl_context,
backlog=self._backlog,
)
class NamedPipeSite(BaseSite):
__slots__ = ("_path",)
def __init__(
self, runner: "BaseRunner", path: str, *, shutdown_timeout: float = 60.0
) -> None:
loop = asyncio.get_event_loop()
if not isinstance(
loop, asyncio.ProactorEventLoop # type: ignore[attr-defined]
):
raise RuntimeError(
"Named Pipes only available in proactor loop under windows"
)
super().__init__(runner, shutdown_timeout=shutdown_timeout)
self._path = path
@property
def name(self) -> str:
return self._path
async def start(self) -> None:
await super().start()
loop = asyncio.get_event_loop()
server = self._runner.server
assert server is not None
_server = await loop.start_serving_pipe( # type: ignore[attr-defined]
server, self._path
)
self._server = _server[0]
class SockSite(BaseSite):
__slots__ = ("_sock", "_name")
def __init__(
self,
runner: "BaseRunner",
sock: socket.socket,
*,
shutdown_timeout: float = 60.0,
ssl_context: SSLContext | None = None,
backlog: int = 128,
) -> None:
super().__init__(
runner,
shutdown_timeout=shutdown_timeout,
ssl_context=ssl_context,
backlog=backlog,
)
self._sock = sock
scheme = "https" if self._ssl_context else "http"
if hasattr(socket, "AF_UNIX") and sock.family == socket.AF_UNIX:
name = f"{scheme}://unix:{sock.getsockname()}:"
else:
host, port = sock.getsockname()[:2]
name = str(URL.build(scheme=scheme, host=host, port=port))
self._name = name
@property
def name(self) -> str:
return self._name
async def start(self) -> None:
await super().start()
loop = asyncio.get_event_loop()
server = self._runner.server
assert server is not None
self._server = await loop.create_server(
server, sock=self._sock, ssl=self._ssl_context, backlog=self._backlog
)
class BaseRunner(ABC):
__slots__ = ("_handle_signals", "_kwargs", "_server", "_sites", "_shutdown_timeout")
def __init__(
self,
*,
handle_signals: bool = False,
shutdown_timeout: float = 60.0,
**kwargs: Any,
) -> None:
self._handle_signals = handle_signals
self._kwargs = kwargs
self._server: Server | None = None
self._sites: list[BaseSite] = []
self._shutdown_timeout = shutdown_timeout
@property
def server(self) -> Server | None:
return self._server
@property
def addresses(self) -> list[Any]:
ret: list[Any] = []
for site in self._sites:
server = site._server
if server is not None:
sockets = server.sockets # type: ignore[attr-defined]
if sockets is not None:
for sock in sockets:
ret.append(sock.getsockname())
return ret
@property
def sites(self) -> set[BaseSite]:
return set(self._sites)
async def setup(self) -> None:
loop = asyncio.get_event_loop()
if self._handle_signals:
try:
loop.add_signal_handler(signal.SIGINT, _raise_graceful_exit)
loop.add_signal_handler(signal.SIGTERM, _raise_graceful_exit)
except NotImplementedError: # pragma: no cover
# add_signal_handler is not implemented on Windows
pass
self._server = await self._make_server()
@abstractmethod
async def shutdown(self) -> None:
"""Call any shutdown hooks to help server close gracefully."""
async def cleanup(self) -> None:
# The loop over sites is intentional, an exception on gather()
# leaves self._sites in unpredictable state.
# The loop guaranties that a site is either deleted on success or
# still present on failure
for site in list(self._sites):
await site.stop()
if self._server: # If setup succeeded
# Yield to event loop to ensure incoming requests prior to stopping the sites
# have all started to be handled before we proceed to close idle connections.
await asyncio.sleep(0)
self._server.pre_shutdown()
await self.shutdown()
await self._server.shutdown(self._shutdown_timeout)
await self._cleanup_server()
self._server = None
if self._handle_signals:
loop = asyncio.get_running_loop()
try:
loop.remove_signal_handler(signal.SIGINT)
loop.remove_signal_handler(signal.SIGTERM)
except NotImplementedError: # pragma: no cover
# remove_signal_handler is not implemented on Windows
pass
@abstractmethod
async def _make_server(self) -> Server:
pass # pragma: no cover
@abstractmethod
async def _cleanup_server(self) -> None:
pass # pragma: no cover
def _reg_site(self, site: BaseSite) -> None:
if site in self._sites:
raise RuntimeError(f"Site {site} is already registered in runner {self}")
self._sites.append(site)
def _check_site(self, site: BaseSite) -> None:
if site not in self._sites:
raise RuntimeError(f"Site {site} is not registered in runner {self}")
def _unreg_site(self, site: BaseSite) -> None:
if site not in self._sites:
raise RuntimeError(f"Site {site} is not registered in runner {self}")
self._sites.remove(site)
class ServerRunner(BaseRunner):
"""Low-level web server runner"""
__slots__ = ("_web_server",)
def __init__(
self, web_server: Server, *, handle_signals: bool = False, **kwargs: Any
) -> None:
super().__init__(handle_signals=handle_signals, **kwargs)
self._web_server = web_server
async def shutdown(self) -> None:
pass
async def _make_server(self) -> Server:
return self._web_server
async def _cleanup_server(self) -> None:
pass
class AppRunner(BaseRunner):
"""Web Application runner"""
__slots__ = ("_app",)
def __init__(
self,
app: Application,
*,
handle_signals: bool = False,
access_log_class: type[AbstractAccessLogger] = AccessLogger,
**kwargs: Any,
) -> None:
super().__init__(handle_signals=handle_signals, **kwargs)
if not isinstance(app, Application):
raise TypeError(
f"The first argument should be web.Application instance, got {app!r}"
)
self._kwargs["access_log_class"] = access_log_class
self._app = app
@property
def app(self) -> Application:
return self._app
async def shutdown(self) -> None:
await self._app.shutdown()
async def _make_server(self) -> Server:
loop = asyncio.get_event_loop()
self._app._set_loop(loop)
self._app.on_startup.freeze()
await self._app.startup()
self._app.freeze()
return self._app._make_handler(loop=loop, **self._kwargs)
async def _cleanup_server(self) -> None:
await self._app.cleanup()
+91
View File
@@ -0,0 +1,91 @@
"""Low level HTTP server."""
import asyncio
from typing import Any, Awaitable, Callable, Dict, List, Optional # noqa
from .abc import AbstractStreamWriter
from .http_parser import RawRequestMessage
from .streams import StreamReader
from .web_protocol import RequestHandler, _RequestFactory, _RequestHandler
from .web_request import BaseRequest
__all__ = ("Server",)
class Server:
def __init__(
self,
handler: _RequestHandler,
*,
request_factory: _RequestFactory | None = None,
handler_cancellation: bool = False,
loop: asyncio.AbstractEventLoop | None = None,
**kwargs: Any,
) -> None:
self._loop = loop or asyncio.get_running_loop()
self._connections: dict[RequestHandler, asyncio.Transport] = {}
self._kwargs = kwargs
# requests_count is the number of requests being processed by the server
# for the lifetime of the server.
self.requests_count = 0
self.request_handler = handler
self.request_factory = request_factory or self._make_request
self.handler_cancellation = handler_cancellation
@property
def connections(self) -> list[RequestHandler]:
return list(self._connections.keys())
def connection_made(
self, handler: RequestHandler, transport: asyncio.Transport
) -> None:
self._connections[handler] = transport
def connection_lost(
self, handler: RequestHandler, exc: BaseException | None = None
) -> None:
if handler in self._connections:
if handler._task_handler:
handler._task_handler.add_done_callback(
lambda f: self._connections.pop(handler, None)
)
else:
del self._connections[handler]
def _make_request(
self,
message: RawRequestMessage,
payload: StreamReader,
protocol: RequestHandler,
writer: AbstractStreamWriter,
task: "asyncio.Task[None]",
) -> BaseRequest:
return BaseRequest(message, payload, protocol, writer, task, self._loop)
def pre_shutdown(self) -> None:
for conn in self._connections:
conn.close()
async def shutdown(self, timeout: float | None = None) -> None:
coros = (conn.shutdown(timeout) for conn in self._connections)
await asyncio.gather(*coros)
self._connections.clear()
def __call__(self) -> RequestHandler:
try:
return RequestHandler(self, loop=self._loop, **self._kwargs)
except TypeError:
# Failsafe creation: remove all custom handler_args
kwargs = {
k: v
for k, v in self._kwargs.items()
if k in ["debug", "access_log_class"]
}
handler = RequestHandler(self, loop=self._loop, **kwargs)
handler.logger.warning(
"Failed to create request handler with custom kwargs %r, "
"falling back to filtered kwargs. This may indicate a "
"misconfiguration.",
self._kwargs,
)
return handler
File diff suppressed because it is too large Load Diff
+783
View File
@@ -0,0 +1,783 @@
import asyncio
import base64
import binascii
import hashlib
import json
import sys
from collections.abc import Callable, Iterable
from typing import Any, Final, Generic, Literal, cast, overload
import attr
from multidict import CIMultiDict
from . import hdrs
from ._websocket.reader import WebSocketDataQueue
from .abc import AbstractStreamWriter
from .client_exceptions import WSMessageTypeError
from .helpers import (
DEFAULT_CHUNK_SIZE,
calculate_timeout_when,
set_exception,
set_result,
)
from .http import (
WS_CLOSED_MESSAGE,
WS_CLOSING_MESSAGE,
WS_KEY,
WebSocketError,
WebSocketReader,
WebSocketWriter,
WSCloseCode,
WSMessage,
WSMessageDecodeText,
WSMessageNoDecodeText,
WSMsgType as WSMsgType,
ws_ext_gen,
ws_ext_parse,
)
from .http_websocket import _INTERNAL_RECEIVE_TYPES
from .log import ws_logger
from .streams import EofStream
from .typedefs import JSONBytesEncoder, JSONDecoder, JSONEncoder
from .web_exceptions import HTTPBadRequest, HTTPException
from .web_request import BaseRequest
from .web_response import StreamResponse
if sys.version_info >= (3, 13):
from typing import TypeVar
else:
from typing_extensions import TypeVar
if sys.version_info >= (3, 12):
from collections.abc import Buffer
else:
from typing import Union
Buffer = Union[bytes, bytearray, "memoryview[int]", "memoryview[bytes]"]
if sys.version_info >= (3, 11):
import asyncio as async_timeout
from typing import Self
else:
import async_timeout
from typing_extensions import Self
__all__ = (
"WebSocketResponse",
"WebSocketReady",
"WSMsgType",
)
THRESHOLD_CONNLOST_ACCESS: Final[int] = 5
# TypeVar for whether text messages are decoded to str (True) or kept as bytes (False)
_DecodeText = TypeVar("_DecodeText", bound=bool, covariant=True, default=Literal[True])
@attr.s(auto_attribs=True, frozen=True, slots=True)
class WebSocketReady:
ok: bool
protocol: str | None
def __bool__(self) -> bool:
return self.ok
class WebSocketResponse(StreamResponse, Generic[_DecodeText]):
_length_check: bool = False
_ws_protocol: str | None = None
_writer: WebSocketWriter | None = None
_reader: WebSocketDataQueue | None = None
_closed: bool = False
_closing: bool = False
_conn_lost: int = 0
_close_code: int | None = None
_loop: asyncio.AbstractEventLoop | None = None
_waiting: bool = False
_close_wait: asyncio.Future[None] | None = None
_exception: BaseException | None = None
_heartbeat_when: float = 0.0
_heartbeat_cb: asyncio.TimerHandle | None = None
_pong_response_cb: asyncio.TimerHandle | None = None
_ping_task: asyncio.Task[None] | None = None
_need_heartbeat_reset: bool = False
_heartbeat_reset_handle: asyncio.Handle | None = None
def __init__(
self,
*,
timeout: float = 10.0,
receive_timeout: float | None = None,
autoclose: bool = True,
autoping: bool = True,
heartbeat: float | None = None,
protocols: Iterable[str] = (),
compress: bool = True,
max_msg_size: int = 4 * 1024 * 1024,
writer_limit: int = DEFAULT_CHUNK_SIZE,
decode_text: bool = True,
) -> None:
super().__init__(status=101)
self._protocols = protocols
self._timeout = timeout
self._receive_timeout = receive_timeout
self._autoclose = autoclose
self._autoping = autoping
self._heartbeat = heartbeat
if heartbeat is not None:
self._pong_heartbeat = heartbeat / 2.0
self._compress: bool | int = compress
self._max_msg_size = max_msg_size
self._writer_limit = writer_limit
self._decode_text = decode_text
self._need_heartbeat_reset = False
self._heartbeat_reset_handle = None
def _cancel_heartbeat(self) -> None:
self._cancel_pong_response_cb()
if self._heartbeat_reset_handle is not None:
self._heartbeat_reset_handle.cancel()
self._heartbeat_reset_handle = None
self._need_heartbeat_reset = False
if self._heartbeat_cb is not None:
self._heartbeat_cb.cancel()
self._heartbeat_cb = None
if self._ping_task is not None:
self._ping_task.cancel()
self._ping_task = None
def _cancel_pong_response_cb(self) -> None:
if self._pong_response_cb is not None:
self._pong_response_cb.cancel()
self._pong_response_cb = None
def _on_data_received(self) -> None:
if self._heartbeat is None or self._need_heartbeat_reset:
return
loop = self._loop
assert loop is not None
# Coalesce multiple chunks received in the same loop tick into a single
# heartbeat reset. Resetting immediately per chunk increases timer churn.
self._need_heartbeat_reset = True
self._heartbeat_reset_handle = loop.call_soon(self._flush_heartbeat_reset)
def _flush_heartbeat_reset(self) -> None:
self._heartbeat_reset_handle = None
if not self._need_heartbeat_reset:
return
self._reset_heartbeat()
self._need_heartbeat_reset = False
def _reset_heartbeat(self) -> None:
if self._heartbeat is None:
return
self._cancel_pong_response_cb()
req = self._req
timeout_ceil_threshold = (
req._protocol._timeout_ceil_threshold if req is not None else 5
)
loop = self._loop
assert loop is not None
now = loop.time()
when = calculate_timeout_when(now, self._heartbeat, timeout_ceil_threshold)
self._heartbeat_when = when
if self._heartbeat_cb is None:
# We do not cancel the previous heartbeat_cb here because
# it generates a significant amount of TimerHandle churn
# which causes asyncio to rebuild the heap frequently.
# Instead _send_heartbeat() will reschedule the next
# heartbeat if it fires too early.
self._heartbeat_cb = loop.call_at(when, self._send_heartbeat)
def _send_heartbeat(self) -> None:
self._heartbeat_cb = None
# If heartbeat reset is pending (data is being received), skip sending
# the ping and let the reset callback handle rescheduling the heartbeat.
if self._need_heartbeat_reset:
return
loop = self._loop
assert loop is not None and self._writer is not None
now = loop.time()
if now < self._heartbeat_when:
# Heartbeat fired too early, reschedule
self._heartbeat_cb = loop.call_at(
self._heartbeat_when, self._send_heartbeat
)
return
req = self._req
timeout_ceil_threshold = (
req._protocol._timeout_ceil_threshold if req is not None else 5
)
when = calculate_timeout_when(now, self._pong_heartbeat, timeout_ceil_threshold)
self._cancel_pong_response_cb()
self._pong_response_cb = loop.call_at(when, self._pong_not_received)
coro = self._writer.send_frame(b"", WSMsgType.PING)
if sys.version_info >= (3, 12):
# Optimization for Python 3.12, try to send the ping
# immediately to avoid having to schedule
# the task on the event loop.
ping_task = asyncio.Task(coro, loop=loop, eager_start=True)
else:
ping_task = loop.create_task(coro)
if not ping_task.done():
self._ping_task = ping_task
ping_task.add_done_callback(self._ping_task_done)
else:
self._ping_task_done(ping_task)
def _ping_task_done(self, task: "asyncio.Task[None]") -> None:
"""Callback for when the ping task completes."""
if not task.cancelled() and (exc := task.exception()):
self._handle_ping_pong_exception(exc)
self._ping_task = None
def _pong_not_received(self) -> None:
if self._req is not None and self._req.transport is not None:
self._handle_ping_pong_exception(
asyncio.TimeoutError(
f"No PONG received after {self._pong_heartbeat} seconds"
)
)
def _handle_ping_pong_exception(self, exc: BaseException) -> None:
"""Handle exceptions raised during ping/pong processing."""
if self._closed:
return
self._set_closed()
self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE)
self._exception = exc
if self._waiting and not self._closing and self._reader is not None:
self._reader.feed_data(WSMessage(WSMsgType.ERROR, exc, None), 0)
def _set_closed(self) -> None:
"""Set the connection to closed.
Cancel any heartbeat timers and set the closed flag.
"""
self._closed = True
self._cancel_heartbeat()
async def prepare(self, request: BaseRequest) -> AbstractStreamWriter:
# make pre-check to don't hide it by do_handshake() exceptions
if self._payload_writer is not None:
return self._payload_writer
protocol, writer = self._pre_start(request)
payload_writer = await super().prepare(request)
assert payload_writer is not None
self._post_start(request, protocol, writer)
await payload_writer.drain()
return payload_writer
def _handshake(
self, request: BaseRequest
) -> tuple["CIMultiDict[str]", str | None, int, bool]:
headers = request.headers
if "websocket" != headers.get(hdrs.UPGRADE, "").lower().strip():
raise HTTPBadRequest(
text=(
f"No WebSocket UPGRADE hdr: {headers.get(hdrs.UPGRADE)}\n Can "
'"Upgrade" only to "WebSocket".'
)
)
if not request._message.upgrade:
raise HTTPBadRequest(
text=f"No CONNECTION upgrade hdr: {headers.get(hdrs.CONNECTION)}"
)
# find common sub-protocol between client and server
protocol: str | None = None
if hdrs.SEC_WEBSOCKET_PROTOCOL in headers:
req_protocols = [
str(proto.strip())
for proto in headers[hdrs.SEC_WEBSOCKET_PROTOCOL].split(",")
]
for proto in req_protocols:
if proto in self._protocols:
protocol = proto
break
else:
# No overlap found: Return no protocol as per spec
ws_logger.warning(
"%s: Client protocols %r don’t overlap server-known ones %r",
request.remote,
req_protocols,
self._protocols,
)
# check supported version
version = headers.get(hdrs.SEC_WEBSOCKET_VERSION, "")
if version not in ("13", "8", "7"):
raise HTTPBadRequest(text=f"Unsupported version: {version}")
# check client handshake for validity
key = headers.get(hdrs.SEC_WEBSOCKET_KEY)
try:
if not key or len(base64.b64decode(key)) != 16:
raise HTTPBadRequest(text=f"Handshake error: {key!r}")
except binascii.Error:
raise HTTPBadRequest(text=f"Handshake error: {key!r}") from None
accept_val = base64.b64encode(
hashlib.sha1(key.encode() + WS_KEY).digest()
).decode()
response_headers = CIMultiDict(
{
hdrs.UPGRADE: "websocket",
hdrs.CONNECTION: "upgrade",
hdrs.SEC_WEBSOCKET_ACCEPT: accept_val,
}
)
notakeover = False
compress = 0
if self._compress:
extensions = headers.get(hdrs.SEC_WEBSOCKET_EXTENSIONS)
# Server side always get return with no exception.
# If something happened, just drop compress extension
compress, notakeover = ws_ext_parse(extensions, isserver=True)
if compress:
enabledext = ws_ext_gen(
compress=compress, isserver=True, server_notakeover=notakeover
)
response_headers[hdrs.SEC_WEBSOCKET_EXTENSIONS] = enabledext
if protocol:
response_headers[hdrs.SEC_WEBSOCKET_PROTOCOL] = protocol
return (
response_headers,
protocol,
compress,
notakeover,
)
def _pre_start(self, request: BaseRequest) -> tuple[str | None, WebSocketWriter]:
self._loop = request._loop
headers, protocol, compress, notakeover = self._handshake(request)
self.set_status(101)
self.headers.update(headers)
self.force_close()
self._compress = compress
transport = request._protocol.transport
if transport is None:
raise ConnectionResetError("Connection lost")
writer = WebSocketWriter(
request._protocol,
transport,
compress=compress,
notakeover=notakeover,
limit=self._writer_limit,
)
return protocol, writer
def _post_start(
self, request: BaseRequest, protocol: str | None, writer: WebSocketWriter
) -> None:
self._ws_protocol = protocol
self._writer = writer
self._reset_heartbeat()
loop = self._loop
assert loop is not None
self._reader = WebSocketDataQueue(
request._protocol, DEFAULT_CHUNK_SIZE, loop=loop
)
parser = WebSocketReader(
self._reader,
self._max_msg_size,
compress=bool(self._compress),
decode_text=self._decode_text,
)
cb = None if self._heartbeat is None else self._on_data_received
request.protocol.set_parser(parser, data_received_cb=cb)
# disable HTTP keepalive for WebSocket
request.protocol.keep_alive(False)
def can_prepare(self, request: BaseRequest) -> WebSocketReady:
if self._writer is not None:
raise RuntimeError("Already started")
try:
_, protocol, _, _ = self._handshake(request)
except HTTPException:
return WebSocketReady(False, None)
else:
return WebSocketReady(True, protocol)
@property
def prepared(self) -> bool:
return self._writer is not None
@property
def closed(self) -> bool:
return self._closed
@property
def close_code(self) -> int | None:
return self._close_code
@property
def ws_protocol(self) -> str | None:
return self._ws_protocol
@property
def compress(self) -> int | bool:
return self._compress
def get_extra_info(self, name: str, default: Any = None) -> Any:
"""Get optional transport information.
If no value associated with ``name`` is found, ``default`` is returned.
"""
writer = self._writer
if writer is None:
return default
transport = writer.transport
if transport is None:
return default
return transport.get_extra_info(name, default)
def exception(self) -> BaseException | None:
return self._exception
async def ping(self, message: bytes = b"") -> None:
if self._writer is None:
raise RuntimeError("Call .prepare() first")
await self._writer.send_frame(message, WSMsgType.PING)
async def pong(self, message: bytes = b"") -> None:
# unsolicited pong
if self._writer is None:
raise RuntimeError("Call .prepare() first")
await self._writer.send_frame(message, WSMsgType.PONG)
async def send_frame(
self, message: bytes, opcode: WSMsgType, compress: int | None = None
) -> None:
"""Send a frame over the websocket."""
if self._writer is None:
raise RuntimeError("Call .prepare() first")
await self._writer.send_frame(message, opcode, compress)
async def send_str(self, data: str, compress: int | None = None) -> None:
if self._writer is None:
raise RuntimeError("Call .prepare() first")
if not isinstance(data, str):
raise TypeError("data argument must be str (%r)" % type(data))
await self._writer.send_frame(
data.encode("utf-8"), WSMsgType.TEXT, compress=compress
)
async def send_bytes(self, data: bytes, compress: int | None = None) -> None:
if self._writer is None:
raise RuntimeError("Call .prepare() first")
if not isinstance(data, (bytes, bytearray, memoryview)):
raise TypeError("data argument must be byte-ish (%r)" % type(data))
await self._writer.send_frame(data, WSMsgType.BINARY, compress=compress)
async def send_json(
self,
data: Any,
compress: int | None = None,
*,
dumps: JSONEncoder = json.dumps,
) -> None:
await self.send_str(dumps(data), compress=compress)
async def send_json_bytes(
self,
data: Any,
compress: int | None = None,
*,
dumps: JSONBytesEncoder,
) -> None:
"""Send JSON data using a bytes-returning encoder as a binary frame.
Use this when your JSON encoder (like orjson) returns bytes
instead of str, avoiding the encode/decode overhead.
"""
await self.send_bytes(dumps(data), compress=compress)
async def write_eof(self) -> None: # type: ignore[override]
if self._eof_sent:
return
if self._payload_writer is None:
raise RuntimeError("Response has not been started")
await self.close()
self._eof_sent = True
async def close(
self, *, code: int = WSCloseCode.OK, message: bytes = b"", drain: bool = True
) -> bool:
"""Close websocket connection."""
if self._writer is None:
raise RuntimeError("Call .prepare() first")
if self._closed:
return False
self._set_closed()
try:
await self._writer.close(code, message)
writer = self._payload_writer
assert writer is not None
if drain:
await writer.drain()
except (asyncio.CancelledError, asyncio.TimeoutError):
self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE)
raise
except Exception as exc:
self._exception = exc
self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE)
return True
reader = self._reader
assert reader is not None
# we need to break `receive()` cycle before we can call
# `reader.read()` as `close()` may be called from different task
if self._waiting:
assert self._loop is not None
assert self._close_wait is None
self._close_wait = self._loop.create_future()
reader.feed_data(WS_CLOSING_MESSAGE, 0)
await self._close_wait
if self._closing:
self._close_transport()
return True
try:
async with async_timeout.timeout(self._timeout):
while True:
msg = await reader.read()
if msg.type is WSMsgType.CLOSE:
self._set_code_close_transport(msg.data)
return True
except asyncio.CancelledError:
self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE)
raise
except Exception as exc:
self._exception = exc
self._set_code_close_transport(WSCloseCode.ABNORMAL_CLOSURE)
return True
def _set_closing(self, code: WSCloseCode) -> None:
"""Set the close code and mark the connection as closing."""
self._closing = True
self._close_code = code
self._cancel_heartbeat()
def _set_code_close_transport(self, code: WSCloseCode) -> None:
"""Set the close code and close the transport."""
self._close_code = code
self._close_transport()
def _close_transport(self) -> None:
"""Close the transport."""
if self._req is not None and self._req.transport is not None:
self._req.transport.close()
@overload
async def receive(
self: "WebSocketResponse[Literal[True]]", timeout: float | None = None
) -> WSMessageDecodeText: ...
@overload
async def receive(
self: "WebSocketResponse[Literal[False]]", timeout: float | None = None
) -> WSMessageNoDecodeText: ...
@overload
async def receive(
self: "WebSocketResponse[_DecodeText]", timeout: float | None = None
) -> WSMessageDecodeText | WSMessageNoDecodeText: ...
async def receive(
self, timeout: float | None = None
) -> WSMessageDecodeText | WSMessageNoDecodeText:
if self._reader is None:
raise RuntimeError("Call .prepare() first")
receive_timeout = timeout or self._receive_timeout
while True:
if self._waiting:
raise RuntimeError("Concurrent call to receive() is not allowed")
if self._closed:
self._conn_lost += 1
if self._conn_lost >= THRESHOLD_CONNLOST_ACCESS:
raise RuntimeError("WebSocket connection is closed.")
return WS_CLOSED_MESSAGE
elif self._closing:
return WS_CLOSING_MESSAGE
try:
self._waiting = True
try:
if receive_timeout:
# Entering the context manager and creating
# Timeout() object can take almost 50% of the
# run time in this loop so we avoid it if
# there is no read timeout.
async with async_timeout.timeout(receive_timeout):
msg = await self._reader.read()
else:
msg = await self._reader.read()
finally:
self._waiting = False
if self._close_wait:
set_result(self._close_wait, None)
except asyncio.TimeoutError:
raise
except EofStream:
self._close_code = WSCloseCode.OK
await self.close()
return WSMessage(WSMsgType.CLOSED, None, None)
except WebSocketError as exc:
self._close_code = exc.code
await self.close(code=exc.code)
return WSMessage(WSMsgType.ERROR, exc, None)
except Exception as exc:
self._exception = exc
self._set_closing(WSCloseCode.ABNORMAL_CLOSURE)
await self.close()
return WSMessage(WSMsgType.ERROR, exc, None)
if msg.type not in _INTERNAL_RECEIVE_TYPES:
# If its not a close/closing/ping/pong message
# we can return it immediately
return msg
if msg.type is WSMsgType.CLOSE:
self._set_closing(msg.data)
# Could be closed while awaiting reader.
if not self._closed and self._autoclose:
# The client is likely going to close the
# connection out from under us so we do not
# want to drain any pending writes as it will
# likely result writing to a broken pipe.
await self.close(drain=False)
elif msg.type is WSMsgType.CLOSING:
self._set_closing(WSCloseCode.OK)
elif msg.type is WSMsgType.PING and self._autoping:
await self.pong(msg.data)
continue
elif msg.type is WSMsgType.PONG and self._autoping:
continue
return msg
@overload
async def receive_str(
self: "WebSocketResponse[Literal[True]]", *, timeout: float | None = None
) -> str: ...
@overload
async def receive_str(
self: "WebSocketResponse[Literal[False]]", *, timeout: float | None = None
) -> bytes: ...
@overload
async def receive_str(
self: "WebSocketResponse[_DecodeText]", *, timeout: float | None = None
) -> str | bytes: ...
async def receive_str(self, *, timeout: float | None = None) -> str | bytes:
"""Receive TEXT message.
Returns str when decode_text=True (default), bytes when decode_text=False.
"""
msg = await self.receive(timeout)
if msg.type is not WSMsgType.TEXT:
raise WSMessageTypeError(
f"Received message {msg.type}:{msg.data!r} is not WSMsgType.TEXT"
)
return cast(str, msg.data)
async def receive_bytes(self, *, timeout: float | None = None) -> bytes:
msg = await self.receive(timeout)
if msg.type is not WSMsgType.BINARY:
raise WSMessageTypeError(
f"Received message {msg.type}:{msg.data!r} is not WSMsgType.BINARY"
)
return cast(bytes, msg.data)
@overload
async def receive_json(
self: "WebSocketResponse[Literal[True]]",
*,
loads: JSONDecoder = ...,
timeout: float | None = None,
) -> Any: ...
@overload
async def receive_json(
self: "WebSocketResponse[Literal[False]]",
*,
loads: Callable[[bytes], Any] = ...,
timeout: float | None = None,
) -> Any: ...
@overload
async def receive_json(
self: "WebSocketResponse[_DecodeText]",
*,
loads: JSONDecoder | Callable[[bytes], Any] = ...,
timeout: float | None = None,
) -> Any: ...
async def receive_json(
self,
*,
loads: JSONDecoder | Callable[[bytes], Any] = json.loads,
timeout: float | None = None,
) -> Any:
data = await self.receive_str(timeout=timeout)
return loads(data) # type: ignore[arg-type]
async def write(self, data: Buffer) -> None:
raise RuntimeError("Cannot call .write() for websocket")
def __aiter__(self) -> Self:
return self
@overload
async def __anext__(
self: "WebSocketResponse[Literal[True]]",
) -> WSMessageDecodeText: ...
@overload
async def __anext__(
self: "WebSocketResponse[Literal[False]]",
) -> WSMessageNoDecodeText: ...
@overload
async def __anext__(
self: "WebSocketResponse[_DecodeText]",
) -> WSMessageDecodeText | WSMessageNoDecodeText: ...
async def __anext__(self) -> WSMessageDecodeText | WSMessageNoDecodeText:
msg = await self.receive()
if msg.type in (WSMsgType.CLOSE, WSMsgType.CLOSING, WSMsgType.CLOSED):
raise StopAsyncIteration
return msg
def _cancel(self, exc: BaseException) -> None:
# web_protocol calls this from connection_lost
# or when the server is shutting down.
self._closing = True
self._cancel_heartbeat()
if self._reader is not None:
set_exception(self._reader, exc)
+278
View File
@@ -0,0 +1,278 @@
"""Async gunicorn worker for aiohttp.web"""
import asyncio
import inspect
import os
import re
import signal
import sys
from types import FrameType
from typing import TYPE_CHECKING, Any, Optional
from gunicorn.config import AccessLogFormat as GunicornAccessLogFormat
from gunicorn.workers import base
from aiohttp import web
from .helpers import set_result
from .web_app import Application
from .web_log import AccessLogger
if TYPE_CHECKING:
import ssl
SSLContext = ssl.SSLContext
else:
try:
import ssl
SSLContext = ssl.SSLContext
except ImportError: # pragma: no cover
ssl = None # type: ignore[assignment]
SSLContext = object # type: ignore[misc,assignment]
__all__ = ("GunicornWebWorker", "GunicornUVLoopWebWorker")
class GunicornWebWorker(base.Worker): # type: ignore[misc,no-any-unimported]
DEFAULT_AIOHTTP_LOG_FORMAT = AccessLogger.LOG_FORMAT
DEFAULT_GUNICORN_LOG_FORMAT = GunicornAccessLogFormat.default
def __init__(self, *args: Any, **kw: Any) -> None: # pragma: no cover
super().__init__(*args, **kw)
self._task: asyncio.Task[None] | None = None
self.exit_code = 0
self._notify_waiter: asyncio.Future[bool] | None = None
def init_process(self) -> None:
# create new event_loop after fork
try:
asyncio.get_event_loop().close()
except RuntimeError:
# No loop was running
pass
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
super().init_process()
def run(self) -> None:
# base.Worker.init_process() sets self.booted = True before
# invoking run(), but for the aiohttp worker the real boot work
# (factory call, runner setup, binding sockets) happens here.
# Reset until _run() reaches the serve loop so that the arbiter
# can tell a startup failure from a normal worker exit and
# halt instead of endlessly respawning workers.
self.booted = False
self._task = self.loop.create_task(self._run())
try:
self.loop.run_until_complete(self._task)
finally:
self.loop.run_until_complete(self.loop.shutdown_asyncgens())
self.loop.close()
sys.exit(self.exit_code)
async def _run(self) -> None:
runner = None
if isinstance(self.wsgi, Application):
app = self.wsgi
elif inspect.iscoroutinefunction(self.wsgi) or (
sys.version_info < (3, 14) and asyncio.iscoroutinefunction(self.wsgi)
):
wsgi = await self.wsgi()
if isinstance(wsgi, web.AppRunner):
runner = wsgi
app = runner.app
else:
app = wsgi
else:
raise RuntimeError(
"wsgi app should be either Application or "
f"async function returning Application, got {self.wsgi}"
)
if runner is None:
access_log = self.log.access_log if self.cfg.accesslog else None
runner = web.AppRunner(
app,
logger=self.log,
keepalive_timeout=self.cfg.keepalive,
access_log=access_log,
access_log_format=self._get_valid_log_format(
self.cfg.access_log_format
),
shutdown_timeout=self.cfg.graceful_timeout / 100 * 95,
)
await runner.setup()
ctx = self._create_ssl_context(self.cfg) if self.cfg.is_ssl else None
runner = runner
assert runner is not None
server = runner.server
assert server is not None
for sock in self.sockets:
site = web.SockSite(
runner,
sock,
ssl_context=ctx,
)
await site.start()
# Sockets are bound; tell the arbiter the worker is ready to
# accept requests. Any failure before this point propagates out
# of run() with self.booted=False so the arbiter exits with
# WORKER_BOOT_ERROR instead of treating this as a clean exit.
self.booted = True
# If our parent changed then we shut down.
pid = os.getpid()
try:
while self.alive: # type: ignore[has-type]
self.notify()
cnt = server.requests_count
if self.max_requests and cnt > self.max_requests:
self.alive = False
self.log.info("Max requests, shutting down: %s", self)
elif pid == os.getpid() and self.ppid != os.getppid():
self.alive = False
self.log.info("Parent changed, shutting down: %s", self)
else:
await self._wait_next_notify()
except Exception:
pass
await runner.cleanup()
def _wait_next_notify(self) -> "asyncio.Future[bool]":
self._notify_waiter_done()
loop = self.loop
assert loop is not None
self._notify_waiter = waiter = loop.create_future()
self.loop.call_later(1.0, self._notify_waiter_done, waiter)
return waiter
def _notify_waiter_done(
self, waiter: Optional["asyncio.Future[bool]"] = None
) -> None:
if waiter is None:
waiter = self._notify_waiter
if waiter is not None:
set_result(waiter, True)
if waiter is self._notify_waiter:
self._notify_waiter = None
def init_signals(self) -> None:
# Set up signals through the event loop API.
self.loop.add_signal_handler(
signal.SIGQUIT, self.handle_quit, signal.SIGQUIT, None
)
self.loop.add_signal_handler(
signal.SIGTERM, self.handle_exit, signal.SIGTERM, None
)
self.loop.add_signal_handler(
signal.SIGINT, self.handle_quit, signal.SIGINT, None
)
self.loop.add_signal_handler(
signal.SIGWINCH, self.handle_winch, signal.SIGWINCH, None
)
self.loop.add_signal_handler(
signal.SIGUSR1, self.handle_usr1, signal.SIGUSR1, None
)
self.loop.add_signal_handler(
signal.SIGABRT, self.handle_abort, signal.SIGABRT, None
)
# Don't let SIGTERM and SIGUSR1 disturb active requests
# by interrupting system calls
signal.siginterrupt(signal.SIGTERM, False)
signal.siginterrupt(signal.SIGUSR1, False)
# Reset SIGCHLD to default so Gunicorn doesn't swallow subprocess
# return codes. Without this, workers inherit the master arbiter's
# SIGCHLD handler, causing spurious "Worker exited" errors when
# application code spawns subprocesses.
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
def handle_quit(self, sig: int, frame: FrameType | None) -> None:
self.alive = False
# worker_int callback
self.cfg.worker_int(self)
# wakeup closing process
self._notify_waiter_done()
def handle_abort(self, sig: int, frame: FrameType | None) -> None:
self.alive = False
self.exit_code = 1
self.cfg.worker_abort(self)
sys.exit(1)
@staticmethod
def _create_ssl_context(cfg: Any) -> "SSLContext":
"""Creates SSLContext instance for usage in asyncio.create_server.
See ssl.SSLSocket.__init__ for more details.
"""
if ssl is None: # pragma: no cover
raise RuntimeError("SSL is not supported.")
ctx = ssl.SSLContext(cfg.ssl_version)
ctx.load_cert_chain(cfg.certfile, cfg.keyfile)
ctx.verify_mode = cfg.cert_reqs
if cfg.ca_certs:
ctx.load_verify_locations(cfg.ca_certs)
if cfg.ciphers:
ctx.set_ciphers(cfg.ciphers)
return ctx
def _get_valid_log_format(self, source_format: str) -> str:
if source_format == self.DEFAULT_GUNICORN_LOG_FORMAT:
return self.DEFAULT_AIOHTTP_LOG_FORMAT
elif re.search(r"%\([^\)]+\)", source_format):
raise ValueError(
"Gunicorn's style options in form of `%(name)s` are not "
"supported for the log formatting. Please use aiohttp's "
"format specification to configure access log formatting: "
"http://docs.aiohttp.org/en/stable/logging.html"
"#format-specification"
)
else:
return source_format
class GunicornUVLoopWebWorker(GunicornWebWorker):
def init_process(self) -> None:
import uvloop
# Close any existing event loop before setting a
# new policy.
try:
asyncio.get_event_loop().close()
except RuntimeError:
# No loop was running
pass
# Setup uvloop policy, so that every
# asyncio.get_event_loop() will create an instance
# of uvloop event loop.
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
super().init_process()
+59
View File
@@ -0,0 +1,59 @@
import sys
from typing import Any, Awaitable, Callable, TypeVar
from frozenlist import FrozenList
if sys.version_info >= (3, 11):
from typing import Unpack
else:
from typing_extensions import Unpack
if sys.version_info >= (3, 13):
from typing import TypeVarTuple
else:
from typing_extensions import TypeVarTuple
_T = TypeVar("_T")
_Ts = TypeVarTuple("_Ts", default=Unpack[tuple[()]])
__version__ = "1.4.0"
__all__ = ("Signal",)
class Signal(FrozenList[Callable[[Unpack[_Ts]], Awaitable[object]]]):
"""Coroutine-based signal implementation.
To connect a callback to a signal, use any list method.
Signals are fired using the send() coroutine, which takes named
arguments.
"""
__slots__ = ("_owner",)
def __init__(self, owner: object):
super().__init__()
self._owner = owner
def __repr__(self) -> str:
return "<Signal owner={}, frozen={}, {!r}>".format(
self._owner, self.frozen, list(self)
)
async def send(self, *args: Unpack[_Ts], **kwargs: Any) -> None:
"""
Sends data to all registered receivers.
"""
if not self.frozen:
raise RuntimeError("Cannot send non-frozen signal.")
for receiver in self:
await receiver(*args, **kwargs)
def __call__(
self, func: Callable[[Unpack[_Ts]], Awaitable[_T]]
) -> Callable[[Unpack[_Ts]], Awaitable[_T]]:
"""Decorator to add a function to this Signal."""
self.append(func)
return func
View File
+79
View File
@@ -0,0 +1,79 @@
# SPDX-License-Identifier: MIT
"""
Argon2 for Python
"""
from . import exceptions, low_level, profiles
from ._legacy import hash_password, hash_password_raw, verify_password
from ._password_hasher import (
DEFAULT_HASH_LENGTH,
DEFAULT_MEMORY_COST,
DEFAULT_PARALLELISM,
DEFAULT_RANDOM_SALT_LENGTH,
DEFAULT_TIME_COST,
PasswordHasher,
)
from ._utils import Parameters, extract_parameters
from .low_level import Type
__title__ = "argon2-cffi"
__author__ = "Hynek Schlawack"
__copyright__ = "Copyright (c) 2015 " + __author__
__license__ = "MIT"
__all__ = [
"DEFAULT_HASH_LENGTH",
"DEFAULT_MEMORY_COST",
"DEFAULT_PARALLELISM",
"DEFAULT_RANDOM_SALT_LENGTH",
"DEFAULT_TIME_COST",
"Parameters",
"PasswordHasher",
"Type",
"exceptions",
"extract_parameters",
"hash_password",
"hash_password_raw",
"low_level",
"profiles",
"verify_password",
]
def __getattr__(name: str) -> str:
dunder_to_metadata = {
"__version__": "version",
"__description__": "summary",
"__uri__": "",
"__url__": "",
"__email__": "",
}
if name not in dunder_to_metadata:
msg = f"module {__name__} has no attribute {name}"
raise AttributeError(msg)
import warnings
from importlib.metadata import metadata
warnings.warn(
f"Accessing argon2.{name} is deprecated and will be "
"removed in a future release. Use importlib.metadata directly "
"to query for argon2-cffi's packaging metadata.",
DeprecationWarning,
stacklevel=2,
)
meta = metadata("argon2-cffi")
if name in ("__uri__", "__url__"):
return meta["Project-URL"].split(" ", 1)[-1]
if name == "__email__":
return meta["Author-email"].split("<", 1)[1].rstrip(">")
return meta[dunder_to_metadata[name]]
+91
View File
@@ -0,0 +1,91 @@
# SPDX-License-Identifier: MIT
from __future__ import annotations
import argparse
import sys
import timeit
from . import (
DEFAULT_HASH_LENGTH,
DEFAULT_MEMORY_COST,
DEFAULT_PARALLELISM,
DEFAULT_TIME_COST,
PasswordHasher,
profiles,
)
def main(argv: list[str]) -> None:
parser = argparse.ArgumentParser(
description="Benchmark Argon2.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
"-n", type=int, default=100, help="Number of iterations to measure."
)
parser.add_argument(
"-t", type=int, help="`time_cost`", default=DEFAULT_TIME_COST
)
parser.add_argument(
"-m", type=int, help="`memory_cost`", default=DEFAULT_MEMORY_COST
)
parser.add_argument(
"-p", type=int, help="`parallelism`", default=DEFAULT_PARALLELISM
)
parser.add_argument(
"-l", type=int, help="`hash_length`", default=DEFAULT_HASH_LENGTH
)
parser.add_argument(
"--profile",
type=str,
help="A profile from `argon2.profiles. Takes precedence.",
default=None,
)
args = parser.parse_args(argv[1:])
password = b"secret"
if args.profile:
ph = PasswordHasher.from_parameters(
getattr(profiles, args.profile.upper())
)
else:
ph = PasswordHasher(
time_cost=args.t,
memory_cost=args.m,
parallelism=args.p,
hash_len=args.l,
)
hash = ph.hash(password)
print(f"Running Argon2id {args.n} times with:")
for name, value, units in [
("hash_len", ph.hash_len, "bytes"),
("memory_cost", ph.memory_cost, "KiB"),
("parallelism", ph.parallelism, "threads"),
("time_cost", ph.time_cost, "iterations"),
]:
print(f"{name}: {value} {units}")
print("\nMeasuring...")
duration = timeit.timeit(
f"ph.verify({hash!r}, {password!r})",
setup=f"""\
from argon2 import PasswordHasher
ph = PasswordHasher(
time_cost={args.t!r},
memory_cost={args.m!r},
parallelism={args.p!r},
hash_len={args.l!r},
)
gc.enable()""",
number=args.n,
)
print(f"\n{duration / args.n * 1000:.1f}ms per password verification")
if __name__ == "__main__": # pragma: no cover
main(sys.argv)
+92
View File
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: MIT
"""
Legacy mid-level functions.
"""
from __future__ import annotations
import os
import warnings
from typing import Literal
from ._password_hasher import (
DEFAULT_HASH_LENGTH,
DEFAULT_MEMORY_COST,
DEFAULT_PARALLELISM,
DEFAULT_RANDOM_SALT_LENGTH,
DEFAULT_TIME_COST,
)
from .low_level import Type, hash_secret, hash_secret_raw, verify_secret
_INSTEAD = " is deprecated, use argon2.PasswordHasher instead"
def hash_password(
password: bytes,
salt: bytes | None = None,
time_cost: int = DEFAULT_TIME_COST,
memory_cost: int = DEFAULT_MEMORY_COST,
parallelism: int = DEFAULT_PARALLELISM,
hash_len: int = DEFAULT_HASH_LENGTH,
type: Type = Type.I,
) -> bytes:
"""
Legacy alias for :func:`argon2.low_level.hash_secret` with default
parameters.
.. deprecated:: 16.0.0
Use :class:`argon2.PasswordHasher` for passwords.
"""
warnings.warn(
"argon2.hash_password" + _INSTEAD, DeprecationWarning, stacklevel=2
)
if salt is None:
salt = os.urandom(DEFAULT_RANDOM_SALT_LENGTH)
return hash_secret(
password, salt, time_cost, memory_cost, parallelism, hash_len, type
)
def hash_password_raw(
password: bytes,
salt: bytes | None = None,
time_cost: int = DEFAULT_TIME_COST,
memory_cost: int = DEFAULT_MEMORY_COST,
parallelism: int = DEFAULT_PARALLELISM,
hash_len: int = DEFAULT_HASH_LENGTH,
type: Type = Type.I,
) -> bytes:
"""
Legacy alias for :func:`argon2.low_level.hash_secret_raw` with default
parameters.
.. deprecated:: 16.0.0
Use :class:`argon2.PasswordHasher` for passwords.
"""
warnings.warn(
"argon2.hash_password_raw" + _INSTEAD, DeprecationWarning, stacklevel=2
)
if salt is None:
salt = os.urandom(DEFAULT_RANDOM_SALT_LENGTH)
return hash_secret_raw(
password, salt, time_cost, memory_cost, parallelism, hash_len, type
)
def verify_password(
hash: bytes, password: bytes, type: Type = Type.I
) -> Literal[True]:
"""
Legacy alias for :func:`argon2.low_level.verify_secret` with default
parameters.
.. deprecated:: 16.0.0
Use :class:`argon2.PasswordHasher` for passwords.
"""
warnings.warn(
"argon2.verify_password" + _INSTEAD, DeprecationWarning, stacklevel=2
)
return verify_secret(hash, password, type)
+287
View File
@@ -0,0 +1,287 @@
# SPDX-License-Identifier: MIT
from __future__ import annotations
import os
from typing import ClassVar, Literal
from ._utils import (
Parameters,
_check_types,
extract_parameters,
validate_params_for_platform,
)
from .exceptions import InvalidHashError
from .low_level import Type, hash_secret, verify_secret
from .profiles import get_default_parameters
default_params = get_default_parameters()
DEFAULT_RANDOM_SALT_LENGTH = default_params.salt_len
DEFAULT_HASH_LENGTH = default_params.hash_len
DEFAULT_TIME_COST = default_params.time_cost
DEFAULT_MEMORY_COST = default_params.memory_cost
DEFAULT_PARALLELISM = default_params.parallelism
def _ensure_bytes(s: bytes | str, encoding: str) -> bytes:
"""
Ensure *s* is a bytes string. Encode using *encoding* if it isn't.
"""
if isinstance(s, bytes):
return s
return s.encode(encoding)
class PasswordHasher:
r"""
High level class to hash passwords with sensible defaults.
Uses Argon2\ **id** by default and uses a random salt_ for hashing. But it
can verify any type of Argon2 as long as the hash is correctly encoded.
The reason for this being a class is both for convenience to carry
parameters and to verify the parameters only *once*. Any unnecessary
slowdown when hashing is a tangible advantage for a brute-force attacker.
Args:
time_cost:
Defines the amount of computation realized and therefore the
execution time, given in number of iterations.
memory_cost: Defines the memory usage, given in kibibytes_.
parallelism:
Defines the number of parallel threads (*changes* the resulting
hash value).
hash_len: Length of the hash in bytes.
salt_len: Length of random salt to be generated for each password.
encoding:
The Argon2 C library expects bytes. So if :meth:`hash` or
:meth:`verify` are passed a ``str``, it will be encoded using this
encoding.
type:
Argon2 type to use. Only change for interoperability with legacy
systems.
.. versionadded:: 16.0.0
.. versionchanged:: 18.2.0
Switch from Argon2i to Argon2id based on the recommendation by the
current RFC draft. See also :doc:`parameters`.
.. versionchanged:: 18.2.0
Changed default *memory_cost* to 100 MiB and default *parallelism* to 8.
.. versionchanged:: 18.2.0 ``verify`` now will determine the type of hash.
.. versionchanged:: 18.3.0 The Argon2 type is configurable now.
.. versionadded:: 21.2.0 :meth:`from_parameters`
.. versionchanged:: 21.2.0
Changed defaults to :data:`argon2.profiles.RFC_9106_LOW_MEMORY`.
.. _salt: https://en.wikipedia.org/wiki/Salt_(cryptography)
.. _kibibytes: https://en.wikipedia.org/wiki/Binary_prefix#kibi
"""
__slots__ = ["_parameters", "encoding"]
_parameters: Parameters
encoding: str
def __init__(
self,
time_cost: int = DEFAULT_TIME_COST,
memory_cost: int = DEFAULT_MEMORY_COST,
parallelism: int = DEFAULT_PARALLELISM,
hash_len: int = DEFAULT_HASH_LENGTH,
salt_len: int = DEFAULT_RANDOM_SALT_LENGTH,
encoding: str = "utf-8",
type: Type = Type.ID,
):
e = _check_types(
time_cost=(time_cost, int),
memory_cost=(memory_cost, int),
parallelism=(parallelism, int),
hash_len=(hash_len, int),
salt_len=(salt_len, int),
encoding=(encoding, str),
type=(type, Type),
)
if e:
raise TypeError(e)
params = Parameters(
type=type,
version=19,
salt_len=salt_len,
hash_len=hash_len,
time_cost=time_cost,
memory_cost=memory_cost,
parallelism=parallelism,
)
validate_params_for_platform(params)
# Cache a Parameters object for check_needs_rehash.
self._parameters = params
self.encoding = encoding
@classmethod
def from_parameters(cls, params: Parameters) -> PasswordHasher:
"""
Construct a `PasswordHasher` from *params*.
Returns:
A `PasswordHasher` instance with the parameters from *params*.
.. versionadded:: 21.2.0
"""
return cls(
time_cost=params.time_cost,
memory_cost=params.memory_cost,
parallelism=params.parallelism,
hash_len=params.hash_len,
salt_len=params.salt_len,
type=params.type,
)
@property
def time_cost(self) -> int:
return self._parameters.time_cost
@property
def memory_cost(self) -> int:
return self._parameters.memory_cost
@property
def parallelism(self) -> int:
return self._parameters.parallelism
@property
def hash_len(self) -> int:
return self._parameters.hash_len
@property
def salt_len(self) -> int:
return self._parameters.salt_len
@property
def type(self) -> Type:
return self._parameters.type
def hash(self, password: str | bytes, *, salt: bytes | None = None) -> str:
"""
Hash *password* and return an encoded hash.
Args:
password: Password to hash.
salt:
If None, a random salt is securely created.
.. danger::
You should **not** pass a salt unless you really know what
you are doing.
Raises:
argon2.exceptions.HashingError: If hashing fails.
Returns:
Hashed *password*.
.. versionadded:: 23.1.0 *salt* parameter
"""
return hash_secret(
secret=_ensure_bytes(password, self.encoding),
salt=salt or os.urandom(self.salt_len),
time_cost=self.time_cost,
memory_cost=self.memory_cost,
parallelism=self.parallelism,
hash_len=self.hash_len,
type=self.type,
).decode("ascii")
_header_to_type: ClassVar[dict[bytes, Type]] = {
b"$argon2i$": Type.I,
b"$argon2d$": Type.D,
b"$argon2id": Type.ID,
}
def verify(
self, hash: str | bytes, password: str | bytes
) -> Literal[True]:
"""
Verify that *password* matches *hash*.
.. warning::
It is assumed that the caller is in full control of the hash. No
other parsing than the determination of the hash type is done by
*argon2-cffi*.
Args:
hash: An encoded hash as returned from :meth:`PasswordHasher.hash`.
password: The password to verify.
Raises:
argon2.exceptions.VerifyMismatchError:
If verification fails because *hash* is not valid for
*password*.
argon2.exceptions.VerificationError:
If verification fails for other reasons.
argon2.exceptions.InvalidHashError:
If *hash* is so clearly invalid, that it couldn't be passed to
Argon2.
Returns:
``True`` on success, otherwise an exception is raised.
.. versionchanged:: 16.1.0
Raise :exc:`~argon2.exceptions.VerifyMismatchError` on mismatches
instead of its more generic superclass.
.. versionadded:: 18.2.0 Hash type agility.
"""
hash = _ensure_bytes(hash, "ascii")
try:
hash_type = self._header_to_type[hash[:9]]
except LookupError:
raise InvalidHashError from None
return verify_secret(
hash, _ensure_bytes(password, self.encoding), hash_type
)
def check_needs_rehash(self, hash: str | bytes) -> bool:
"""
Check whether *hash* was created using the instance's parameters.
Whenever your Argon2 parameters -- or *argon2-cffi*'s defaults! --
change, you should rehash your passwords at the next opportunity. The
common approach is to do that whenever a user logs in, since that
should be the only time when you have access to the cleartext
password.
Therefore it's best practice to check -- and if necessary rehash --
passwords after each successful authentication.
Args:
hash: An encoded Argon2 password hash.
Returns:
Whether *hash* was created using the instance's parameters.
.. versionadded:: 18.2.0
.. versionchanged:: 24.1.0 Accepts bytes for *hash*.
"""
if isinstance(hash, bytes):
hash = hash.decode("ascii")
return self._parameters != extract_parameters(hash)
+174
View File
@@ -0,0 +1,174 @@
# SPDX-License-Identifier: MIT
from __future__ import annotations
import platform
import sys
from dataclasses import dataclass
from typing import Any
from .exceptions import InvalidHashError, UnsupportedParametersError
from .low_level import Type
NoneType = type(None)
def _check_types(**kw: Any) -> str | None:
"""
Check each ``name: (value, types)`` in *kw*.
Returns a human-readable string of all violations or `None``.
"""
errors = []
for name, (value, types) in kw.items():
if not isinstance(value, types):
if isinstance(types, tuple):
types = ", or ".join(t.__name__ for t in types)
else:
types = types.__name__
errors.append(
f"'{name}' must be a {types} (got {type(value).__name__})"
)
if errors != []:
return ", ".join(errors) + "."
return None
def _is_wasm() -> bool:
return sys.platform == "emscripten" or platform.machine() in [
"wasm32",
"wasm64",
]
def _decoded_str_len(length: int) -> int:
"""
Compute how long an encoded string of length *l* becomes.
"""
rem = length % 4
if rem == 3:
last_group_len = 2
elif rem == 2:
last_group_len = 1
else:
last_group_len = 0
return length // 4 * 3 + last_group_len
@dataclass
class Parameters:
"""
Argon2 hash parameters.
See :doc:`parameters` on how to pick them.
Attributes:
type: Hash type.
version: Argon2 version.
salt_len: Length of the salt in bytes.
hash_len: Length of the hash in bytes.
time_cost: Time cost in iterations.
memory_cost: Memory cost in kibibytes.
parallelism: Number of parallel threads.
.. versionadded:: 18.2.0
"""
type: Type
version: int
salt_len: int
hash_len: int
time_cost: int
memory_cost: int
parallelism: int
__slots__ = (
"hash_len",
"memory_cost",
"parallelism",
"salt_len",
"time_cost",
"type",
"version",
)
_NAME_TO_TYPE = {"argon2id": Type.ID, "argon2i": Type.I, "argon2d": Type.D}
_REQUIRED_KEYS = sorted(("v", "m", "t", "p"))
def extract_parameters(hash: str) -> Parameters:
"""
Extract parameters from an encoded *hash*.
Args:
hash: An encoded Argon2 hash string.
Returns:
The parameters used to create the hash.
.. versionadded:: 18.2.0
"""
parts = hash.split("$")
# Backwards compatibility for Argon v1.2 hashes
if len(parts) == 5:
parts.insert(2, "v=18")
if len(parts) != 6:
raise InvalidHashError
if parts[0]:
raise InvalidHashError
try:
type = _NAME_TO_TYPE[parts[1]]
kvs = {
k: int(v)
for k, v in (
s.split("=") for s in [parts[2], *parts[3].split(",")]
)
}
except Exception: # noqa: BLE001
raise InvalidHashError from None
if sorted(kvs.keys()) != _REQUIRED_KEYS:
raise InvalidHashError
return Parameters(
type=type,
salt_len=_decoded_str_len(len(parts[4])),
hash_len=_decoded_str_len(len(parts[5])),
version=kvs["v"],
time_cost=kvs["t"],
memory_cost=kvs["m"],
parallelism=kvs["p"],
)
def validate_params_for_platform(params: Parameters) -> None:
"""
Validate *params* against current platform.
Args:
params: Parameters to be validated
Returns:
None
"""
if _is_wasm() and params.parallelism != 1:
msg = "In WebAssembly environments `parallelism` must be 1."
raise UnsupportedParametersError(msg)
+66
View File
@@ -0,0 +1,66 @@
# SPDX-License-Identifier: MIT
from __future__ import annotations
class Argon2Error(Exception):
"""
Superclass of all argon2 exceptions.
Never thrown directly.
"""
class VerificationError(Argon2Error):
"""
Verification failed.
You can find the original error message from Argon2 in ``args[0]``.
"""
class VerifyMismatchError(VerificationError):
"""
The secret does not match the hash.
Subclass of :exc:`argon2.exceptions.VerificationError`.
.. versionadded:: 16.1.0
"""
class HashingError(Argon2Error):
"""
Raised if hashing failed.
You can find the original error message from Argon2 in ``args[0]``.
"""
class InvalidHashError(ValueError):
"""
Raised if the hash is invalid before passing it to Argon2.
.. versionadded:: 23.1.0
As a replacement for :exc:`argon2.exceptions.InvalidHash`.
"""
class UnsupportedParametersError(ValueError):
"""
Raised if the current platform does not support the parameters.
For example, in WebAssembly parallelism must be set to 1.
.. versionadded:: 25.1.0
"""
InvalidHash = InvalidHashError
"""
Deprecated alias for :class:`InvalidHashError`.
.. versionadded:: 18.2.0
.. deprecated:: 23.1.0
Use :exc:`argon2.exceptions.InvalidHashError` instead.
"""
+253
View File
@@ -0,0 +1,253 @@
# SPDX-License-Identifier: MIT
"""
Low-level functions if you want to build your own higher level abstractions.
.. warning::
This is a "Hazardous Materials" module. You should **ONLY** use it if
you're 100% absolutely sure that you know what you're doing because this
module is full of land mines, dragons, and dinosaurs with laser guns.
"""
from __future__ import annotations
from enum import Enum
from typing import Any, Literal
from _argon2_cffi_bindings import ffi, lib
from .exceptions import HashingError, VerificationError, VerifyMismatchError
__all__ = [
"ARGON2_VERSION",
"Type",
"ffi",
"hash_secret",
"hash_secret_raw",
"verify_secret",
]
ARGON2_VERSION = lib.ARGON2_VERSION_NUMBER
"""
The latest version of the Argon2 algorithm that is supported (and used by
default).
.. versionadded:: 16.1.0
"""
class Type(Enum):
"""
Enum of Argon2 variants.
Please see :doc:`parameters` on how to pick one.
"""
D = lib.Argon2_d
I = lib.Argon2_i # noqa: E741
ID = lib.Argon2_id
def hash_secret(
secret: bytes,
salt: bytes,
time_cost: int,
memory_cost: int,
parallelism: int,
hash_len: int,
type: Type,
version: int = ARGON2_VERSION,
) -> bytes:
"""
Hash *secret* and return an **encoded** hash.
An encoded hash can be directly passed into :func:`verify_secret` as it
contains all parameters and the salt.
Args:
secret: Secret to hash.
salt: A salt_. Should be random and different for each secret.
type: Which Argon2 variant to use.
version: Which Argon2 version to use.
For an explanation of the Argon2 parameters see
:class:`argon2.PasswordHasher`.
Returns:
An encoded Argon2 hash.
Raises:
argon2.exceptions.HashingError: If hashing fails.
.. versionadded:: 16.0.0
.. _salt: https://en.wikipedia.org/wiki/Salt_(cryptography)
"""
size = (
lib.argon2_encodedlen(
time_cost,
memory_cost,
parallelism,
len(salt),
hash_len,
type.value,
)
+ 1
)
buf = ffi.new("char[]", size)
rv = lib.argon2_hash(
time_cost,
memory_cost,
parallelism,
ffi.new("uint8_t[]", secret),
len(secret),
ffi.new("uint8_t[]", salt),
len(salt),
ffi.NULL,
hash_len,
buf,
size,
type.value,
version,
)
if rv != lib.ARGON2_OK:
raise HashingError(error_to_str(rv))
return ffi.string(buf) # type: ignore[no-any-return]
def hash_secret_raw(
secret: bytes,
salt: bytes,
time_cost: int,
memory_cost: int,
parallelism: int,
hash_len: int,
type: Type,
version: int = ARGON2_VERSION,
) -> bytes:
"""
Hash *password* and return a **raw** hash.
This function takes the same parameters as :func:`hash_secret`.
.. versionadded:: 16.0.0
"""
buf = ffi.new("uint8_t[]", hash_len)
rv = lib.argon2_hash(
time_cost,
memory_cost,
parallelism,
ffi.new("uint8_t[]", secret),
len(secret),
ffi.new("uint8_t[]", salt),
len(salt),
buf,
hash_len,
ffi.NULL,
0,
type.value,
version,
)
if rv != lib.ARGON2_OK:
raise HashingError(error_to_str(rv))
return bytes(ffi.buffer(buf, hash_len))
def verify_secret(hash: bytes, secret: bytes, type: Type) -> Literal[True]:
"""
Verify whether *secret* is correct for *hash* of *type*.
Args:
hash:
An encoded Argon2 hash as returned by :func:`hash_secret`.
secret:
The secret to verify whether it matches the one in *hash*.
type: Type for *hash*.
Raises:
argon2.exceptions.VerifyMismatchError:
If verification fails because *hash* is not valid for *secret* of
*type*.
argon2.exceptions.VerificationError:
If verification fails for other reasons.
Returns:
``True`` on success, raise :exc:`~argon2.exceptions.VerificationError`
otherwise.
.. versionadded:: 16.0.0
.. versionchanged:: 16.1.0
Raise :exc:`~argon2.exceptions.VerifyMismatchError` on mismatches
instead of its more generic superclass.
"""
rv = lib.argon2_verify(
ffi.new("char[]", hash),
ffi.new("uint8_t[]", secret),
len(secret),
type.value,
)
if rv == lib.ARGON2_OK:
return True
if rv == lib.ARGON2_VERIFY_MISMATCH:
raise VerifyMismatchError(error_to_str(rv))
raise VerificationError(error_to_str(rv))
def core(context: Any, type: int) -> int:
"""
Direct binding to the ``argon2_ctx`` function.
.. warning::
This is a strictly advanced function working on raw C data structures.
Both Argon2's and *argon2-cffi*'s higher-level bindings do a lot of
sanity checks and housekeeping work that *you* are now responsible for
(e.g. clearing buffers). The structure of the *context* object can,
has, and will change with *any* release!
Use at your own peril; *argon2-cffi* does *not* use this binding
itself.
Args:
context:
A CFFI Argon2 context object (i.e. an ``struct Argon2_Context`` /
``argon2_context``).
type:
Which Argon2 variant to use. You can use the ``value`` field of
:class:`Type`'s fields.
Returns:
An Argon2 error code. Can be transformed into a string using
:func:`error_to_str`.
.. versionadded:: 16.0.0
"""
return lib.argon2_ctx(context, type) # type: ignore[no-any-return]
def error_to_str(error: int) -> str:
"""
Convert an Argon2 error code into a native string.
Args:
error: An Argon2 error code as returned by :func:`core`.
Returns:
A human-readable string describing the error.
.. versionadded:: 16.0.0
"""
return ffi.string(lib.argon2_error_message(error)).decode("ascii") # type: ignore[no-any-return]
+79
View File
@@ -0,0 +1,79 @@
# SPDX-License-Identifier: MIT
"""
This module offers access to standardized parameters that you can load using
:meth:`argon2.PasswordHasher.from_parameters()`. See the `source code
<https://github.com/hynek/argon2-cffi/blob/main/src/argon2/profiles.py>`_ for
concrete values and :doc:`parameters` for more information.
.. versionadded:: 21.2.0
"""
from __future__ import annotations
import dataclasses
from ._utils import Parameters, _is_wasm
from .low_level import Type
def get_default_parameters() -> Parameters:
"""
Create default parameters for current platform.
Returns:
Default, compatible, parameters for current platform.
.. versionadded:: 25.1.0
"""
params = RFC_9106_LOW_MEMORY
if _is_wasm():
params = dataclasses.replace(params, parallelism=1)
return params
# FIRST RECOMMENDED option per RFC 9106.
RFC_9106_HIGH_MEMORY = Parameters(
type=Type.ID,
version=19,
salt_len=16,
hash_len=32,
time_cost=1,
memory_cost=2097152, # 2 GiB
parallelism=4,
)
# SECOND RECOMMENDED option per RFC 9106.
RFC_9106_LOW_MEMORY = Parameters(
type=Type.ID,
version=19,
salt_len=16,
hash_len=32,
time_cost=3,
memory_cost=65536, # 64 MiB
parallelism=4,
)
# The pre-RFC defaults in argon2-cffi 18.2.0 - 21.1.0.
PRE_21_2 = Parameters(
type=Type.ID,
version=19,
salt_len=16,
hash_len=16,
time_cost=2,
memory_cost=102400, # 100 MiB
parallelism=8,
)
# Only for testing!
CHEAPEST = Parameters(
type=Type.ID,
version=19,
salt_len=8,
hash_len=4,
time_cost=1,
memory_cost=8,
parallelism=1,
)
View File
+276
View File
@@ -0,0 +1,276 @@
import asyncio
import enum
import sys
from types import TracebackType
from typing import Optional, Type, final
__version__ = "5.0.1"
__all__ = ("timeout", "timeout_at", "Timeout")
def timeout(delay: Optional[float]) -> "Timeout":
"""timeout context manager.
Useful in cases when you want to apply timeout logic around block
of code or in cases when asyncio.wait_for is not suitable. For example:
>>> async with timeout(0.001):
... async with aiohttp.get('https://github.com') as r:
... await r.text()
delay - value in seconds or None to disable timeout logic
"""
loop = asyncio.get_running_loop()
if delay is not None:
deadline = loop.time() + delay # type: Optional[float]
else:
deadline = None
return Timeout(deadline, loop)
def timeout_at(deadline: Optional[float]) -> "Timeout":
"""Schedule the timeout at absolute time.
deadline argument points on the time in the same clock system
as loop.time().
Please note: it is not POSIX time but a time with
undefined starting base, e.g. the time of the system power on.
>>> async with timeout_at(loop.time() + 10):
... async with aiohttp.get('https://github.com') as r:
... await r.text()
"""
loop = asyncio.get_running_loop()
return Timeout(deadline, loop)
class _State(enum.Enum):
INIT = "INIT"
ENTER = "ENTER"
TIMEOUT = "TIMEOUT"
EXIT = "EXIT"
if sys.version_info >= (3, 11):
class _Expired:
__slots__ = ("_val",)
def __init__(self, val: bool) -> None:
self._val = val
def __call__(self) -> bool:
return self._val
def __bool__(self) -> bool:
return self._val
def __repr__(self) -> str:
return repr(self._val)
def __str__(self) -> str:
return str(self._val)
@final
class Timeout(asyncio.Timeout): # type: ignore[misc]
# Supports full asyncio.Timeout API.
# Also provides several asyncio_timeout specific methods
# for backward compatibility.
def __init__(
self, deadline: Optional[float], loop: asyncio.AbstractEventLoop
) -> None:
super().__init__(deadline)
@property
def expired(self) -> _Expired:
# a hacky property hat can provide both roles:
# timeout.expired() from asyncio
# timeout.expired from asyncio_timeout
return _Expired(super().expired())
@property
def deadline(self) -> Optional[float]:
return self.when()
def reject(self) -> None:
"""Reject scheduled timeout if any."""
# cancel is maybe better name but
# task.cancel() raises CancelledError in asyncio world.
self.reschedule(None)
def shift(self, delay: float) -> None:
"""Advance timeout on delay seconds.
The delay can be negative.
Raise RuntimeError if shift is called when deadline is not scheduled
"""
deadline = self.when()
if deadline is None:
raise RuntimeError("cannot shift timeout if deadline is not scheduled")
self.reschedule(deadline + delay)
def update(self, deadline: float) -> None:
"""Set deadline to absolute value.
deadline argument points on the time in the same clock system
as loop.time().
If new deadline is in the past the timeout is raised immediately.
Please note: it is not POSIX time but a time with
undefined starting base, e.g. the time of the system power on.
"""
self.reschedule(deadline)
else:
@final
class Timeout:
# Internal class, please don't instantiate it directly
# Use timeout() and timeout_at() public factories instead.
#
# Implementation note: `async with timeout()` is preferred
# over `with timeout()`.
# While technically the Timeout class implementation
# doesn't need to be async at all,
# the `async with` statement explicitly points that
# the context manager should be used from async function context.
#
# This design allows to avoid many silly misusages.
#
# TimeoutError is raised immediately when scheduled
# if the deadline is passed.
# The purpose is to time out as soon as possible
# without waiting for the next await expression.
__slots__ = ("_deadline", "_loop", "_state", "_timeout_handler", "_task")
def __init__(
self, deadline: Optional[float], loop: asyncio.AbstractEventLoop
) -> None:
self._loop = loop
self._state = _State.INIT
self._task: Optional["asyncio.Task[object]"] = None
self._timeout_handler = None # type: Optional[asyncio.Handle]
if deadline is None:
self._deadline = None # type: Optional[float]
else:
self.update(deadline)
async def __aenter__(self) -> "Timeout":
self._do_enter()
return self
async def __aexit__(
self,
exc_type: Optional[Type[BaseException]],
exc_val: Optional[BaseException],
exc_tb: Optional[TracebackType],
) -> Optional[bool]:
self._do_exit(exc_type)
return None
@property
def expired(self) -> bool:
"""Is timeout expired during execution?"""
return self._state == _State.TIMEOUT
@property
def deadline(self) -> Optional[float]:
return self._deadline
def reject(self) -> None:
"""Reject scheduled timeout if any."""
# cancel is maybe better name but
# task.cancel() raises CancelledError in asyncio world.
if self._state not in (_State.INIT, _State.ENTER):
raise RuntimeError(f"invalid state {self._state.value}")
self._reject()
def _reject(self) -> None:
self._task = None
if self._timeout_handler is not None:
self._timeout_handler.cancel()
self._timeout_handler = None
def shift(self, delay: float) -> None:
"""Advance timeout on delay seconds.
The delay can be negative.
Raise RuntimeError if shift is called when deadline is not scheduled
"""
deadline = self._deadline
if deadline is None:
raise RuntimeError("cannot shift timeout if deadline is not scheduled")
self.update(deadline + delay)
def update(self, deadline: float) -> None:
"""Set deadline to absolute value.
deadline argument points on the time in the same clock system
as loop.time().
If new deadline is in the past the timeout is raised immediately.
Please note: it is not POSIX time but a time with
undefined starting base, e.g. the time of the system power on.
"""
if self._state == _State.EXIT:
raise RuntimeError("cannot reschedule after exit from context manager")
if self._state == _State.TIMEOUT:
raise RuntimeError("cannot reschedule expired timeout")
if self._timeout_handler is not None:
self._timeout_handler.cancel()
self._deadline = deadline
if self._state != _State.INIT:
self._reschedule()
def _reschedule(self) -> None:
assert self._state == _State.ENTER
deadline = self._deadline
if deadline is None:
return
now = self._loop.time()
if self._timeout_handler is not None:
self._timeout_handler.cancel()
self._task = asyncio.current_task()
if deadline <= now:
self._timeout_handler = self._loop.call_soon(self._on_timeout)
else:
self._timeout_handler = self._loop.call_at(deadline, self._on_timeout)
def _do_enter(self) -> None:
if self._state != _State.INIT:
raise RuntimeError(f"invalid state {self._state.value}")
self._state = _State.ENTER
self._reschedule()
def _do_exit(self, exc_type: Optional[Type[BaseException]]) -> None:
if exc_type is asyncio.CancelledError and self._state == _State.TIMEOUT:
assert self._task is not None
self._timeout_handler = None
self._task = None
raise asyncio.TimeoutError
# timeout has not expired
self._state = _State.EXIT
self._reject()
return None
def _on_timeout(self) -> None:
assert self._task is not None
self._task.cancel()
self._state = _State.TIMEOUT
# drop the reference early
self._timeout_handler = None
+1
View File
@@ -0,0 +1 @@
Placeholder
+104
View File
@@ -0,0 +1,104 @@
# SPDX-License-Identifier: MIT
"""
Classes Without Boilerplate
"""
from functools import partial
from typing import Callable, Literal, Protocol
from . import converters, exceptions, filters, setters, validators
from ._cmp import cmp_using
from ._config import get_run_validators, set_run_validators
from ._funcs import asdict, assoc, astuple, has, resolve_types
from ._make import (
NOTHING,
Attribute,
Converter,
Factory,
_Nothing,
attrib,
attrs,
evolve,
fields,
fields_dict,
make_class,
validate,
)
from ._next_gen import define, field, frozen, mutable
from ._version_info import VersionInfo
s = attributes = attrs
ib = attr = attrib
dataclass = partial(attrs, auto_attribs=True) # happy Easter ;)
class AttrsInstance(Protocol):
pass
NothingType = Literal[_Nothing.NOTHING]
__all__ = [
"NOTHING",
"Attribute",
"AttrsInstance",
"Converter",
"Factory",
"NothingType",
"asdict",
"assoc",
"astuple",
"attr",
"attrib",
"attributes",
"attrs",
"cmp_using",
"converters",
"define",
"evolve",
"exceptions",
"field",
"fields",
"fields_dict",
"filters",
"frozen",
"get_run_validators",
"has",
"ib",
"make_class",
"mutable",
"resolve_types",
"s",
"set_run_validators",
"setters",
"validate",
"validators",
]
def _make_getattr(mod_name: str) -> Callable:
"""
Create a metadata proxy for packaging information that uses *mod_name* in
its warnings and errors.
"""
def __getattr__(name: str) -> str:
if name not in ("__version__", "__version_info__"):
msg = f"module {mod_name} has no attribute {name}"
raise AttributeError(msg)
from importlib.metadata import metadata
meta = metadata("attrs")
if name == "__version_info__":
return VersionInfo._from_version_string(meta["version"])
return meta["version"]
return __getattr__
__getattr__ = _make_getattr(__name__)
+389
View File
@@ -0,0 +1,389 @@
import enum
import sys
from typing import (
Any,
Callable,
Generic,
Literal,
Mapping,
Protocol,
Sequence,
TypeVar,
overload,
)
# `import X as X` is required to make these public
from . import converters as converters
from . import exceptions as exceptions
from . import filters as filters
from . import setters as setters
from . import validators as validators
from ._cmp import cmp_using as cmp_using
from ._typing_compat import AttrsInstance_
from ._version_info import VersionInfo
from attrs import (
define as define,
field as field,
mutable as mutable,
frozen as frozen,
_EqOrderType,
_ValidatorType,
_ConverterType,
_ReprArgType,
_OnSetAttrType,
_OnSetAttrArgType,
_FieldTransformer,
_ValidatorArgType,
)
if sys.version_info >= (3, 10):
from typing import TypeGuard, TypeAlias
else:
from typing_extensions import TypeGuard, TypeAlias
if sys.version_info >= (3, 11):
from typing import dataclass_transform
else:
from typing_extensions import dataclass_transform
__version__: str
__version_info__: VersionInfo
__title__: str
__description__: str
__url__: str
__uri__: str
__author__: str
__email__: str
__license__: str
__copyright__: str
_T = TypeVar("_T")
_C = TypeVar("_C", bound=type)
_FilterType = Callable[["Attribute[_T]", _T], bool]
# We subclass this here to keep the protocol's qualified name clean.
class AttrsInstance(AttrsInstance_, Protocol):
pass
_A = TypeVar("_A", bound=type[AttrsInstance])
class _Nothing(enum.Enum):
NOTHING = enum.auto()
NOTHING = _Nothing.NOTHING
NothingType: TypeAlias = Literal[_Nothing.NOTHING]
# NOTE: Factory lies about its return type to make this possible:
# `x: List[int] # = Factory(list)`
# Work around mypy issue #4554 in the common case by using an overload.
@overload
def Factory(factory: Callable[[], _T]) -> _T: ...
@overload
def Factory(
factory: Callable[[Any], _T],
takes_self: Literal[True],
) -> _T: ...
@overload
def Factory(
factory: Callable[[], _T],
takes_self: Literal[False],
) -> _T: ...
In = TypeVar("In")
Out = TypeVar("Out")
class Converter(Generic[In, Out]):
@overload
def __init__(self, converter: Callable[[In], Out]) -> None: ...
@overload
def __init__(
self,
converter: Callable[[In, AttrsInstance, Attribute], Out],
*,
takes_self: Literal[True],
takes_field: Literal[True],
) -> None: ...
@overload
def __init__(
self,
converter: Callable[[In, Attribute], Out],
*,
takes_field: Literal[True],
) -> None: ...
@overload
def __init__(
self,
converter: Callable[[In, AttrsInstance], Out],
*,
takes_self: Literal[True],
) -> None: ...
class Attribute(Generic[_T]):
name: str
default: _T | None
validator: _ValidatorType[_T] | None
repr: _ReprArgType
cmp: _EqOrderType
eq: _EqOrderType
order: _EqOrderType
hash: bool | None
init: bool
converter: Converter | None
metadata: dict[Any, Any]
type: type[_T] | None
kw_only: bool
on_setattr: _OnSetAttrType
alias: str | None
def evolve(self, **changes: Any) -> "Attribute[Any]": ...
# NOTE: We had several choices for the annotation to use for type arg:
# 1) Type[_T]
# - Pros: Handles simple cases correctly
# - Cons: Might produce less informative errors in the case of conflicting
# TypeVars e.g. `attr.ib(default='bad', type=int)`
# 2) Callable[..., _T]
# - Pros: Better error messages than #1 for conflicting TypeVars
# - Cons: Terrible error messages for validator checks.
# e.g. attr.ib(type=int, validator=validate_str)
# -> error: Cannot infer function type argument
# 3) type (and do all of the work in the mypy plugin)
# - Pros: Simple here, and we could customize the plugin with our own errors.
# - Cons: Would need to write mypy plugin code to handle all the cases.
# We chose option #1.
# `attr` lies about its return type to make the following possible:
# attr() -> Any
# attr(8) -> int
# attr(validator=<some callable>) -> Whatever the callable expects.
# This makes this type of assignments possible:
# x: int = attr(8)
#
# This form catches explicit None or no default but with no other arguments
# returns Any.
@overload
def attrib(
default: None = ...,
validator: None = ...,
repr: _ReprArgType = ...,
cmp: _EqOrderType | None = ...,
hash: bool | None = ...,
init: bool = ...,
metadata: Mapping[Any, Any] | None = ...,
type: None = ...,
converter: None = ...,
factory: None = ...,
kw_only: bool | None = ...,
eq: _EqOrderType | None = ...,
order: _EqOrderType | None = ...,
on_setattr: _OnSetAttrArgType | None = ...,
alias: str | None = ...,
) -> Any: ...
# This form catches an explicit None or no default and infers the type from the
# other arguments.
@overload
def attrib(
default: None = ...,
validator: _ValidatorArgType[_T] | None = ...,
repr: _ReprArgType = ...,
cmp: _EqOrderType | None = ...,
hash: bool | None = ...,
init: bool = ...,
metadata: Mapping[Any, Any] | None = ...,
type: type[_T] | None = ...,
converter: _ConverterType
| list[_ConverterType]
| tuple[_ConverterType]
| None = ...,
factory: Callable[[], _T] | None = ...,
kw_only: bool | None = ...,
eq: _EqOrderType | None = ...,
order: _EqOrderType | None = ...,
on_setattr: _OnSetAttrArgType | None = ...,
alias: str | None = ...,
) -> _T: ...
# This form catches an explicit default argument.
@overload
def attrib(
default: _T,
validator: _ValidatorArgType[_T] | None = ...,
repr: _ReprArgType = ...,
cmp: _EqOrderType | None = ...,
hash: bool | None = ...,
init: bool = ...,
metadata: Mapping[Any, Any] | None = ...,
type: type[_T] | None = ...,
converter: _ConverterType
| list[_ConverterType]
| tuple[_ConverterType]
| None = ...,
factory: Callable[[], _T] | None = ...,
kw_only: bool | None = ...,
eq: _EqOrderType | None = ...,
order: _EqOrderType | None = ...,
on_setattr: _OnSetAttrArgType | None = ...,
alias: str | None = ...,
) -> _T: ...
# This form covers type=non-Type: e.g. forward references (str), Any
@overload
def attrib(
default: _T | None = ...,
validator: _ValidatorArgType[_T] | None = ...,
repr: _ReprArgType = ...,
cmp: _EqOrderType | None = ...,
hash: bool | None = ...,
init: bool = ...,
metadata: Mapping[Any, Any] | None = ...,
type: object = ...,
converter: _ConverterType
| list[_ConverterType]
| tuple[_ConverterType]
| None = ...,
factory: Callable[[], _T] | None = ...,
kw_only: bool | None = ...,
eq: _EqOrderType | None = ...,
order: _EqOrderType | None = ...,
on_setattr: _OnSetAttrArgType | None = ...,
alias: str | None = ...,
) -> Any: ...
@overload
@dataclass_transform(order_default=True, field_specifiers=(attrib, field))
def attrs(
maybe_cls: _C,
these: dict[str, Any] | None = ...,
repr_ns: str | None = ...,
repr: bool = ...,
cmp: _EqOrderType | None = ...,
hash: bool | None = ...,
init: bool = ...,
slots: bool = ...,
frozen: bool = ...,
weakref_slot: bool = ...,
str: bool = ...,
auto_attribs: bool = ...,
kw_only: bool = ...,
cache_hash: bool = ...,
auto_exc: bool = ...,
eq: _EqOrderType | None = ...,
order: _EqOrderType | None = ...,
auto_detect: bool = ...,
collect_by_mro: bool = ...,
getstate_setstate: bool | None = ...,
on_setattr: _OnSetAttrArgType | None = ...,
field_transformer: _FieldTransformer | None = ...,
match_args: bool = ...,
unsafe_hash: bool | None = ...,
) -> _C: ...
@overload
@dataclass_transform(order_default=True, field_specifiers=(attrib, field))
def attrs(
maybe_cls: None = ...,
these: dict[str, Any] | None = ...,
repr_ns: str | None = ...,
repr: bool = ...,
cmp: _EqOrderType | None = ...,
hash: bool | None = ...,
init: bool = ...,
slots: bool = ...,
frozen: bool = ...,
weakref_slot: bool = ...,
str: bool = ...,
auto_attribs: bool = ...,
kw_only: bool = ...,
cache_hash: bool = ...,
auto_exc: bool = ...,
eq: _EqOrderType | None = ...,
order: _EqOrderType | None = ...,
auto_detect: bool = ...,
collect_by_mro: bool = ...,
getstate_setstate: bool | None = ...,
on_setattr: _OnSetAttrArgType | None = ...,
field_transformer: _FieldTransformer | None = ...,
match_args: bool = ...,
unsafe_hash: bool | None = ...,
) -> Callable[[_C], _C]: ...
def fields(cls: type[AttrsInstance] | AttrsInstance) -> Any: ...
def fields_dict(cls: type[AttrsInstance]) -> dict[str, Attribute[Any]]: ...
def validate(inst: AttrsInstance) -> None: ...
def resolve_types(
cls: _A,
globalns: dict[str, Any] | None = ...,
localns: dict[str, Any] | None = ...,
attribs: list[Attribute[Any]] | None = ...,
include_extras: bool = ...,
) -> _A: ...
# TODO: add support for returning a proper attrs class from the mypy plugin
# we use Any instead of _CountingAttr so that e.g. `make_class('Foo',
# [attr.ib()])` is valid
def make_class(
name: str,
attrs: list[str] | tuple[str, ...] | dict[str, Any],
bases: tuple[type, ...] = ...,
class_body: dict[str, Any] | None = ...,
repr_ns: str | None = ...,
repr: bool = ...,
cmp: _EqOrderType | None = ...,
hash: bool | None = ...,
init: bool = ...,
slots: bool = ...,
frozen: bool = ...,
weakref_slot: bool = ...,
str: bool = ...,
auto_attribs: bool = ...,
kw_only: bool = ...,
cache_hash: bool = ...,
auto_exc: bool = ...,
eq: _EqOrderType | None = ...,
order: _EqOrderType | None = ...,
collect_by_mro: bool = ...,
on_setattr: _OnSetAttrArgType | None = ...,
field_transformer: _FieldTransformer | None = ...,
) -> type: ...
# _funcs --
# TODO: add support for returning TypedDict from the mypy plugin
# FIXME: asdict/astuple do not honor their factory args. Waiting on one of
# these:
# https://github.com/python/mypy/issues/4236
# https://github.com/python/typing/issues/253
# XXX: remember to fix attrs.asdict/astuple too!
def asdict(
inst: AttrsInstance,
recurse: bool = ...,
filter: _FilterType[Any] | None = ...,
dict_factory: type[Mapping[Any, Any]] = ...,
retain_collection_types: bool = ...,
value_serializer: Callable[[type, Attribute[Any], Any], Any] | None = ...,
tuple_keys: bool | None = ...,
) -> dict[str, Any]: ...
# TODO: add support for returning NamedTuple from the mypy plugin
def astuple(
inst: AttrsInstance,
recurse: bool = ...,
filter: _FilterType[Any] | None = ...,
tuple_factory: type[Sequence[Any]] = ...,
retain_collection_types: bool = ...,
) -> tuple[Any, ...]: ...
def has(cls: type) -> TypeGuard[type[AttrsInstance]]: ...
def assoc(inst: _T, **changes: Any) -> _T: ...
def evolve(inst: _T, **changes: Any) -> _T: ...
# _config --
def set_run_validators(run: bool) -> None: ...
def get_run_validators() -> bool: ...
# aliases --
s = attributes = attrs
ib = attr = attrib
dataclass = attrs # Technically, partial(attrs, auto_attribs=True) ;)
+160
View File
@@ -0,0 +1,160 @@
# SPDX-License-Identifier: MIT
import functools
import types
from ._make import __ne__
_operation_names = {"eq": "==", "lt": "<", "le": "<=", "gt": ">", "ge": ">="}
def cmp_using(
eq=None,
lt=None,
le=None,
gt=None,
ge=None,
require_same_type=True,
class_name="Comparable",
):
"""
Create a class that can be passed into `attrs.field`'s ``eq``, ``order``,
and ``cmp`` arguments to customize field comparison.
The resulting class will have a full set of ordering methods if at least
one of ``{lt, le, gt, ge}`` and ``eq`` are provided.
Args:
eq (typing.Callable | None):
Callable used to evaluate equality of two objects.
lt (typing.Callable | None):
Callable used to evaluate whether one object is less than another
object.
le (typing.Callable | None):
Callable used to evaluate whether one object is less than or equal
to another object.
gt (typing.Callable | None):
Callable used to evaluate whether one object is greater than
another object.
ge (typing.Callable | None):
Callable used to evaluate whether one object is greater than or
equal to another object.
require_same_type (bool):
When `True`, equality and ordering methods will return
`NotImplemented` if objects are not of the same type.
class_name (str | None): Name of class. Defaults to "Comparable".
See `comparison` for more details.
.. versionadded:: 21.1.0
"""
body = {
"__slots__": ["value"],
"__init__": _make_init(),
"_requirements": [],
"_is_comparable_to": _is_comparable_to,
}
# Add operations.
num_order_functions = 0
has_eq_function = False
if eq is not None:
has_eq_function = True
body["__eq__"] = _make_operator("eq", eq)
body["__ne__"] = __ne__
if lt is not None:
num_order_functions += 1
body["__lt__"] = _make_operator("lt", lt)
if le is not None:
num_order_functions += 1
body["__le__"] = _make_operator("le", le)
if gt is not None:
num_order_functions += 1
body["__gt__"] = _make_operator("gt", gt)
if ge is not None:
num_order_functions += 1
body["__ge__"] = _make_operator("ge", ge)
type_ = types.new_class(
class_name, (object,), {}, lambda ns: ns.update(body)
)
# Add same type requirement.
if require_same_type:
type_._requirements.append(_check_same_type)
# Add total ordering if at least one operation was defined.
if 0 < num_order_functions < 4:
if not has_eq_function:
# functools.total_ordering requires __eq__ to be defined,
# so raise early error here to keep a nice stack.
msg = "eq must be define is order to complete ordering from lt, le, gt, ge."
raise ValueError(msg)
type_ = functools.total_ordering(type_)
return type_
def _make_init():
"""
Create __init__ method.
"""
def __init__(self, value):
"""
Initialize object with *value*.
"""
self.value = value
return __init__
def _make_operator(name, func):
"""
Create operator method.
"""
def method(self, other):
if not self._is_comparable_to(other):
return NotImplemented
result = func(self.value, other.value)
if result is NotImplemented:
return NotImplemented
return result
method.__name__ = f"__{name}__"
method.__doc__ = (
f"Return a {_operation_names[name]} b. Computed by attrs."
)
return method
def _is_comparable_to(self, other):
"""
Check whether `other` is comparable to `self`.
"""
return all(func(self, other) for func in self._requirements)
def _check_same_type(self, other):
"""
Return True if *self* and *other* are of the same type, False otherwise.
"""
return other.value.__class__ is self.value.__class__
+13
View File
@@ -0,0 +1,13 @@
from typing import Any, Callable
_CompareWithType = Callable[[Any, Any], bool]
def cmp_using(
eq: _CompareWithType | None = ...,
lt: _CompareWithType | None = ...,
le: _CompareWithType | None = ...,
gt: _CompareWithType | None = ...,
ge: _CompareWithType | None = ...,
require_same_type: bool = ...,
class_name: str = ...,
) -> type: ...
+99
View File
@@ -0,0 +1,99 @@
# SPDX-License-Identifier: MIT
import inspect
import platform
import sys
import threading
from collections.abc import Mapping, Sequence # noqa: F401
from typing import _GenericAlias
PYPY = platform.python_implementation() == "PyPy"
PY_3_10_PLUS = sys.version_info[:2] >= (3, 10)
PY_3_11_PLUS = sys.version_info[:2] >= (3, 11)
PY_3_12_PLUS = sys.version_info[:2] >= (3, 12)
PY_3_13_PLUS = sys.version_info[:2] >= (3, 13)
PY_3_14_PLUS = sys.version_info[:2] >= (3, 14)
if PY_3_14_PLUS:
import annotationlib
# We request forward-ref annotations to not break in the presence of
# forward references.
def _get_annotations(cls):
return annotationlib.get_annotations(
cls, format=annotationlib.Format.FORWARDREF
)
else:
def _get_annotations(cls):
"""
Get annotations for *cls*.
"""
return cls.__dict__.get("__annotations__", {})
class _AnnotationExtractor:
"""
Extract type annotations from a callable, returning None whenever there
is none.
"""
__slots__ = ["sig"]
def __init__(self, callable):
try:
self.sig = inspect.signature(callable)
except (ValueError, TypeError): # inspect failed
self.sig = None
def get_first_param_type(self):
"""
Return the type annotation of the first argument if it's not empty.
"""
if not self.sig:
return None
params = list(self.sig.parameters.values())
if params and params[0].annotation is not inspect.Parameter.empty:
return params[0].annotation
return None
def get_return_type(self):
"""
Return the return type if it's not empty.
"""
if (
self.sig
and self.sig.return_annotation is not inspect.Signature.empty
):
return self.sig.return_annotation
return None
# Thread-local global to track attrs instances which are already being repr'd.
# This is needed because there is no other (thread-safe) way to pass info
# about the instances that are already being repr'd through the call stack
# in order to ensure we don't perform infinite recursion.
#
# For instance, if an instance contains a dict which contains that instance,
# we need to know that we're already repr'ing the outside instance from within
# the dict's repr() call.
#
# This lives here rather than in _make.py so that the functions in _make.py
# don't have a direct reference to the thread-local in their globals dict.
# If they have such a reference, it breaks cloudpickle.
repr_context = threading.local()
def get_generic_base(cl):
"""If this is a generic class (A[str]), return the generic base for it."""
if cl.__class__ is _GenericAlias:
return cl.__origin__
return None
+31
View File
@@ -0,0 +1,31 @@
# SPDX-License-Identifier: MIT
__all__ = ["get_run_validators", "set_run_validators"]
_run_validators = True
def set_run_validators(run):
"""
Set whether or not validators are run. By default, they are run.
.. deprecated:: 21.3.0 It will not be removed, but it also will not be
moved to new ``attrs`` namespace. Use `attrs.validators.set_disabled()`
instead.
"""
if not isinstance(run, bool):
msg = "'run' must be bool."
raise TypeError(msg)
global _run_validators
_run_validators = run
def get_run_validators():
"""
Return whether or not validators are run.
.. deprecated:: 21.3.0 It will not be removed, but it also will not be
moved to new ``attrs`` namespace. Use `attrs.validators.get_disabled()`
instead.
"""
return _run_validators
+497
View File
@@ -0,0 +1,497 @@
# SPDX-License-Identifier: MIT
import copy
from ._compat import get_generic_base
from ._make import _OBJ_SETATTR, NOTHING, fields
from .exceptions import AttrsAttributeNotFoundError
_ATOMIC_TYPES = frozenset(
{
type(None),
bool,
int,
float,
str,
complex,
bytes,
type(...),
type,
range,
property,
}
)
def asdict(
inst,
recurse=True,
filter=None,
dict_factory=dict,
retain_collection_types=False,
value_serializer=None,
):
"""
Return the *attrs* attribute values of *inst* as a dict.
Optionally recurse into other *attrs*-decorated classes.
Args:
inst: Instance of an *attrs*-decorated class.
recurse (bool): Recurse into classes that are also *attrs*-decorated.
filter (~typing.Callable):
A callable whose return code determines whether an attribute or
element is included (`True`) or dropped (`False`). Is called with
the `attrs.Attribute` as the first argument and the value as the
second argument.
dict_factory (~typing.Callable):
A callable to produce dictionaries from. For example, to produce
ordered dictionaries instead of normal Python dictionaries, pass in
``collections.OrderedDict``.
retain_collection_types (bool):
Do not convert to `list` when encountering an attribute whose type
is `tuple` or `set`. Only meaningful if *recurse* is `True`.
value_serializer (typing.Callable | None):
A hook that is called for every attribute or dict key/value. It
receives the current instance, field and value and must return the
(updated) value. The hook is run *after* the optional *filter* has
been applied.
Returns:
Return type of *dict_factory*.
Raises:
attrs.exceptions.NotAnAttrsClassError:
If *cls* is not an *attrs* class.
.. versionadded:: 16.0.0 *dict_factory*
.. versionadded:: 16.1.0 *retain_collection_types*
.. versionadded:: 20.3.0 *value_serializer*
.. versionadded:: 21.3.0
If a dict has a collection for a key, it is serialized as a tuple.
"""
attrs = fields(inst.__class__)
rv = dict_factory()
for a in attrs:
v = getattr(inst, a.name)
if filter is not None and not filter(a, v):
continue
if value_serializer is not None:
v = value_serializer(inst, a, v)
if recurse is True:
value_type = type(v)
if value_type in _ATOMIC_TYPES:
rv[a.name] = v
elif has(value_type):
rv[a.name] = asdict(
v,
recurse=True,
filter=filter,
dict_factory=dict_factory,
retain_collection_types=retain_collection_types,
value_serializer=value_serializer,
)
elif issubclass(value_type, (tuple, list, set, frozenset)):
cf = value_type if retain_collection_types is True else list
items = [
_asdict_anything(
i,
is_key=False,
filter=filter,
dict_factory=dict_factory,
retain_collection_types=retain_collection_types,
value_serializer=value_serializer,
)
for i in v
]
try:
rv[a.name] = cf(items)
except TypeError:
if not issubclass(cf, tuple):
raise
# Workaround for TypeError: cf.__new__() missing 1 required
# positional argument (which appears, for a namedturle)
rv[a.name] = cf(*items)
elif issubclass(value_type, dict):
df = dict_factory
rv[a.name] = df(
(
_asdict_anything(
kk,
is_key=True,
filter=filter,
dict_factory=df,
retain_collection_types=retain_collection_types,
value_serializer=value_serializer,
),
_asdict_anything(
vv,
is_key=False,
filter=filter,
dict_factory=df,
retain_collection_types=retain_collection_types,
value_serializer=value_serializer,
),
)
for kk, vv in v.items()
)
else:
rv[a.name] = v
else:
rv[a.name] = v
return rv
def _asdict_anything(
val,
is_key,
filter,
dict_factory,
retain_collection_types,
value_serializer,
):
"""
``asdict`` only works on attrs instances, this works on anything.
"""
val_type = type(val)
if val_type in _ATOMIC_TYPES:
rv = val
if value_serializer is not None:
rv = value_serializer(None, None, rv)
elif getattr(val_type, "__attrs_attrs__", None) is not None:
# Attrs class.
rv = asdict(
val,
recurse=True,
filter=filter,
dict_factory=dict_factory,
retain_collection_types=retain_collection_types,
value_serializer=value_serializer,
)
elif issubclass(val_type, (tuple, list, set, frozenset)):
if retain_collection_types is True:
cf = val.__class__
elif is_key:
cf = tuple
else:
cf = list
rv = cf(
[
_asdict_anything(
i,
is_key=False,
filter=filter,
dict_factory=dict_factory,
retain_collection_types=retain_collection_types,
value_serializer=value_serializer,
)
for i in val
]
)
elif issubclass(val_type, dict):
df = dict_factory
rv = df(
(
_asdict_anything(
kk,
is_key=True,
filter=filter,
dict_factory=df,
retain_collection_types=retain_collection_types,
value_serializer=value_serializer,
),
_asdict_anything(
vv,
is_key=False,
filter=filter,
dict_factory=df,
retain_collection_types=retain_collection_types,
value_serializer=value_serializer,
),
)
for kk, vv in val.items()
)
else:
rv = val
if value_serializer is not None:
rv = value_serializer(None, None, rv)
return rv
def astuple(
inst,
recurse=True,
filter=None,
tuple_factory=tuple,
retain_collection_types=False,
):
"""
Return the *attrs* attribute values of *inst* as a tuple.
Optionally recurse into other *attrs*-decorated classes.
Args:
inst: Instance of an *attrs*-decorated class.
recurse (bool):
Recurse into classes that are also *attrs*-decorated.
filter (~typing.Callable):
A callable whose return code determines whether an attribute or
element is included (`True`) or dropped (`False`). Is called with
the `attrs.Attribute` as the first argument and the value as the
second argument.
tuple_factory (~typing.Callable):
A callable to produce tuples from. For example, to produce lists
instead of tuples.
retain_collection_types (bool):
Do not convert to `list` or `dict` when encountering an attribute
which type is `tuple`, `dict` or `set`. Only meaningful if
*recurse* is `True`.
Returns:
Return type of *tuple_factory*
Raises:
attrs.exceptions.NotAnAttrsClassError:
If *cls* is not an *attrs* class.
.. versionadded:: 16.2.0
"""
attrs = fields(inst.__class__)
rv = []
retain = retain_collection_types # Very long. :/
for a in attrs:
v = getattr(inst, a.name)
if filter is not None and not filter(a, v):
continue
value_type = type(v)
if recurse is True:
if value_type in _ATOMIC_TYPES:
rv.append(v)
elif has(value_type):
rv.append(
astuple(
v,
recurse=True,
filter=filter,
tuple_factory=tuple_factory,
retain_collection_types=retain,
)
)
elif issubclass(value_type, (tuple, list, set, frozenset)):
cf = v.__class__ if retain is True else list
items = [
(
astuple(
j,
recurse=True,
filter=filter,
tuple_factory=tuple_factory,
retain_collection_types=retain,
)
if has(j.__class__)
else j
)
for j in v
]
try:
rv.append(cf(items))
except TypeError:
if not issubclass(cf, tuple):
raise
# Workaround for TypeError: cf.__new__() missing 1 required
# positional argument (which appears, for a namedturle)
rv.append(cf(*items))
elif issubclass(value_type, dict):
df = value_type if retain is True else dict
rv.append(
df(
(
(
astuple(
kk,
tuple_factory=tuple_factory,
retain_collection_types=retain,
)
if has(kk.__class__)
else kk
),
(
astuple(
vv,
tuple_factory=tuple_factory,
retain_collection_types=retain,
)
if has(vv.__class__)
else vv
),
)
for kk, vv in v.items()
)
)
else:
rv.append(v)
else:
rv.append(v)
return rv if tuple_factory is list else tuple_factory(rv)
def has(cls):
"""
Check whether *cls* is a class with *attrs* attributes.
Args:
cls (type): Class to introspect.
Raises:
TypeError: If *cls* is not a class.
Returns:
bool:
"""
attrs = getattr(cls, "__attrs_attrs__", None)
if attrs is not None:
return True
# No attrs, maybe it's a specialized generic (A[str])?
generic_base = get_generic_base(cls)
if generic_base is not None:
generic_attrs = getattr(generic_base, "__attrs_attrs__", None)
if generic_attrs is not None:
# Stick it on here for speed next time.
cls.__attrs_attrs__ = generic_attrs
return generic_attrs is not None
return False
def assoc(inst, **changes):
"""
Copy *inst* and apply *changes*.
This is different from `evolve` that applies the changes to the arguments
that create the new instance.
`evolve`'s behavior is preferable, but there are `edge cases`_ where it
doesn't work. Therefore `assoc` is deprecated, but will not be removed.
.. _`edge cases`: https://github.com/python-attrs/attrs/issues/251
Args:
inst: Instance of a class with *attrs* attributes.
changes: Keyword changes in the new copy.
Returns:
A copy of inst with *changes* incorporated.
Raises:
attrs.exceptions.AttrsAttributeNotFoundError:
If *attr_name* couldn't be found on *cls*.
attrs.exceptions.NotAnAttrsClassError:
If *cls* is not an *attrs* class.
.. deprecated:: 17.1.0
Use `attrs.evolve` instead if you can. This function will not be
removed du to the slightly different approach compared to
`attrs.evolve`, though.
"""
new = copy.copy(inst)
attrs = fields(inst.__class__)
for k, v in changes.items():
a = getattr(attrs, k, NOTHING)
if a is NOTHING:
msg = f"{k} is not an attrs attribute on {new.__class__}."
raise AttrsAttributeNotFoundError(msg)
_OBJ_SETATTR(new, k, v)
return new
def resolve_types(
cls, globalns=None, localns=None, attribs=None, include_extras=True
):
"""
Resolve any strings and forward annotations in type annotations.
This is only required if you need concrete types in :class:`Attribute`'s
*type* field. In other words, you don't need to resolve your types if you
only use them for static type checking.
With no arguments, names will be looked up in the module in which the class
was created. If this is not what you want, for example, if the name only
exists inside a method, you may pass *globalns* or *localns* to specify
other dictionaries in which to look up these names. See the docs of
`typing.get_type_hints` for more details.
Args:
cls (type): Class to resolve.
globalns (dict | None): Dictionary containing global variables.
localns (dict | None): Dictionary containing local variables.
attribs (list | None):
List of attribs for the given class. This is necessary when calling
from inside a ``field_transformer`` since *cls* is not an *attrs*
class yet.
include_extras (bool):
Resolve more accurately, if possible. Pass ``include_extras`` to
``typing.get_hints``, if supported by the typing module. On
supported Python versions (3.9+), this resolves the types more
accurately.
Raises:
TypeError: If *cls* is not a class.
attrs.exceptions.NotAnAttrsClassError:
If *cls* is not an *attrs* class and you didn't pass any attribs.
NameError: If types cannot be resolved because of missing variables.
Returns:
*cls* so you can use this function also as a class decorator. Please
note that you have to apply it **after** `attrs.define`. That means the
decorator has to come in the line **before** `attrs.define`.
.. versionadded:: 20.1.0
.. versionadded:: 21.1.0 *attribs*
.. versionadded:: 23.1.0 *include_extras*
"""
# Since calling get_type_hints is expensive we cache whether we've
# done it already.
if getattr(cls, "__attrs_types_resolved__", None) != cls:
import typing
kwargs = {
"globalns": globalns,
"localns": localns,
"include_extras": include_extras,
}
hints = typing.get_type_hints(cls, **kwargs)
for field in fields(cls) if attribs is None else attribs:
if field.name in hints:
# Since fields have been frozen we must work around it.
_OBJ_SETATTR(field, "type", hints[field.name])
# We store the class we resolved so that subclasses know they haven't
# been resolved.
cls.__attrs_types_resolved__ = cls
# Return the class so you can use it as a decorator too.
return cls
File diff suppressed because it is too large Load Diff
+674
View File
@@ -0,0 +1,674 @@
# SPDX-License-Identifier: MIT
"""
These are keyword-only APIs that call `attr.s` and `attr.ib` with different
default values.
"""
from functools import partial
from . import setters
from ._funcs import asdict as _asdict
from ._funcs import astuple as _astuple
from ._make import (
_DEFAULT_ON_SETATTR,
NOTHING,
_frozen_setattrs,
attrib,
attrs,
)
from .exceptions import NotAnAttrsClassError, UnannotatedAttributeError
def define(
maybe_cls=None,
*,
these=None,
repr=None,
unsafe_hash=None,
hash=None,
init=None,
slots=True,
frozen=False,
weakref_slot=True,
str=False,
auto_attribs=None,
kw_only=False,
cache_hash=False,
auto_exc=True,
eq=None,
order=False,
auto_detect=True,
getstate_setstate=None,
on_setattr=None,
field_transformer=None,
match_args=True,
force_kw_only=False,
):
r"""
A class decorator that adds :term:`dunder methods` according to
:term:`fields <field>` specified using :doc:`type annotations <types>`,
`field()` calls, or the *these* argument.
Since *attrs* patches or replaces an existing class, you cannot use
`object.__init_subclass__` with *attrs* classes, because it runs too early.
As a replacement, you can define ``__attrs_init_subclass__`` on your class.
It will be called by *attrs* classes that subclass it after they're
created. See also :ref:`init-subclass`.
Args:
slots (bool):
Create a :term:`slotted class <slotted classes>` that's more
memory-efficient. Slotted classes are generally superior to the
default dict classes, but have some gotchas you should know about,
so we encourage you to read the :term:`glossary entry <slotted
classes>`.
auto_detect (bool):
Instead of setting the *init*, *repr*, *eq*, and *hash* arguments
explicitly, assume they are set to True **unless any** of the
involved methods for one of the arguments is implemented in the
*current* class (meaning, it is *not* inherited from some base
class).
So, for example by implementing ``__eq__`` on a class yourself,
*attrs* will deduce ``eq=False`` and will create *neither*
``__eq__`` *nor* ``__ne__`` (but Python classes come with a
sensible ``__ne__`` by default, so it *should* be enough to only
implement ``__eq__`` in most cases).
Passing :data:`True` or :data:`False` to *init*, *repr*, *eq*, or *hash*
overrides whatever *auto_detect* would determine.
auto_exc (bool):
If the class subclasses `BaseException` (which implicitly includes
any subclass of any exception), the following happens to behave
like a well-behaved Python exception class:
- the values for *eq*, *order*, and *hash* are ignored and the
instances compare and hash by the instance's ids [#]_ ,
- all attributes that are either passed into ``__init__`` or have a
default value are additionally available as a tuple in the
``args`` attribute,
- the value of *str* is ignored leaving ``__str__`` to base
classes.
.. [#]
Note that *attrs* will *not* remove existing implementations of
``__hash__`` or the equality methods. It just won't add own
ones.
on_setattr (~typing.Callable | list[~typing.Callable] | None | ~typing.Literal[attrs.setters.NO_OP]):
A callable that is run whenever the user attempts to set an
attribute (either by assignment like ``i.x = 42`` or by using
`setattr` like ``setattr(i, "x", 42)``). It receives the same
arguments as validators: the instance, the attribute that is being
modified, and the new value.
If no exception is raised, the attribute is set to the return value
of the callable.
If a list of callables is passed, they're automatically wrapped in
an `attrs.setters.pipe`.
If left None, the default behavior is to run converters and
validators whenever an attribute is set.
init (bool):
Create a ``__init__`` method that initializes the *attrs*
attributes. Leading underscores are stripped for the argument name,
unless an alias is set on the attribute.
.. seealso::
`init` shows advanced ways to customize the generated
``__init__`` method, including executing code before and after.
repr(bool):
Create a ``__repr__`` method with a human readable representation
of *attrs* attributes.
str (bool):
Create a ``__str__`` method that is identical to ``__repr__``. This
is usually not necessary except for `Exception`\ s.
eq (bool | None):
If True or None (default), add ``__eq__`` and ``__ne__`` methods
that check two instances for equality.
.. seealso::
`comparison` describes how to customize the comparison behavior
going as far comparing NumPy arrays.
order (bool | None):
If True, add ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__``
methods that behave like *eq* above and allow instances to be
ordered.
They compare the instances as if they were tuples of their *attrs*
attributes if and only if the types of both classes are
*identical*.
If `None` mirror value of *eq*.
.. seealso:: `comparison`
unsafe_hash (bool | None):
If None (default), the ``__hash__`` method is generated according
how *eq* and *frozen* are set.
1. If *both* are True, *attrs* will generate a ``__hash__`` for
you.
2. If *eq* is True and *frozen* is False, ``__hash__`` will be set
to None, marking it unhashable (which it is).
3. If *eq* is False, ``__hash__`` will be left untouched meaning
the ``__hash__`` method of the base class will be used. If the
base class is `object`, this means it will fall back to id-based
hashing.
Although not recommended, you can decide for yourself and force
*attrs* to create one (for example, if the class is immutable even
though you didn't freeze it programmatically) by passing True or
not. Both of these cases are rather special and should be used
carefully.
.. seealso::
- Our documentation on `hashing`,
- Python's documentation on `object.__hash__`,
- and the `GitHub issue that led to the default \ behavior
<https://github.com/python-attrs/attrs/issues/136>`_ for more
details.
hash (bool | None):
Deprecated alias for *unsafe_hash*. *unsafe_hash* takes precedence.
cache_hash (bool):
Ensure that the object's hash code is computed only once and stored
on the object. If this is set to True, hashing must be either
explicitly or implicitly enabled for this class. If the hash code
is cached, avoid any reassignments of fields involved in hash code
computation or mutations of the objects those fields point to after
object creation. If such changes occur, the behavior of the
object's hash code is undefined.
frozen (bool):
Make instances immutable after initialization. If someone attempts
to modify a frozen instance, `attrs.exceptions.FrozenInstanceError`
is raised.
.. note::
1. This is achieved by installing a custom ``__setattr__``
method on your class, so you can't implement your own.
2. True immutability is impossible in Python.
3. This *does* have a minor a runtime performance `impact
<how-frozen>` when initializing new instances. In other
words: ``__init__`` is slightly slower with ``frozen=True``.
4. If a class is frozen, you cannot modify ``self`` in
``__attrs_post_init__`` or a self-written ``__init__``. You
can circumvent that limitation by using
``object.__setattr__(self, "attribute_name", value)``.
5. Subclasses of a frozen class are frozen too.
kw_only (bool):
Make attributes keyword-only in the generated ``__init__`` (if
*init* is False, this parameter is ignored). Attributes that
explicitly set ``kw_only=False`` are not affected; base class
attributes are also not affected.
Also see *force_kw_only*.
weakref_slot (bool):
Make instances weak-referenceable. This has no effect unless
*slots* is True.
field_transformer (~typing.Callable | None):
A function that is called with the original class object and all
fields right before *attrs* finalizes the class. You can use this,
for example, to automatically add converters or validators to
fields based on their types.
.. seealso:: `transform-fields`
match_args (bool):
If True (default), set ``__match_args__`` on the class to support
:pep:`634` (*Structural Pattern Matching*). It is a tuple of all
non-keyword-only ``__init__`` parameter names on Python 3.10 and
later. Ignored on older Python versions.
collect_by_mro (bool):
If True, *attrs* collects attributes from base classes correctly
according to the `method resolution order
<https://docs.python.org/3/howto/mro.html>`_. If False, *attrs*
will mimic the (wrong) behavior of `dataclasses` and :pep:`681`.
See also `issue #428
<https://github.com/python-attrs/attrs/issues/428>`_.
force_kw_only (bool):
A back-compat flag for restoring pre-25.4.0 behavior. If True and
``kw_only=True``, all attributes are made keyword-only, including
base class attributes, and those set to ``kw_only=False`` at the
attribute level. Defaults to False.
See also `issue #980
<https://github.com/python-attrs/attrs/issues/980>`_.
getstate_setstate (bool | None):
.. note::
This is usually only interesting for slotted classes and you
should probably just set *auto_detect* to True.
If True, ``__getstate__`` and ``__setstate__`` are generated and
attached to the class. This is necessary for slotted classes to be
pickleable. If left None, it's True by default for slotted classes
and False for dict classes.
If *auto_detect* is True, and *getstate_setstate* is left None, and
**either** ``__getstate__`` or ``__setstate__`` is detected
directly on the class (meaning: not inherited), it is set to False
(this is usually what you want).
auto_attribs (bool | None):
If True, look at type annotations to determine which attributes to
use, like `dataclasses`. If False, it will only look for explicit
:func:`field` class attributes, like classic *attrs*.
If left None, it will guess:
1. If any attributes are annotated and no unannotated
`attrs.field`\ s are found, it assumes *auto_attribs=True*.
2. Otherwise it assumes *auto_attribs=False* and tries to collect
`attrs.field`\ s.
If *attrs* decides to look at type annotations, **all** fields
**must** be annotated. If *attrs* encounters a field that is set to
a :func:`field` / `attr.ib` but lacks a type annotation, an
`attrs.exceptions.UnannotatedAttributeError` is raised. Use
``field_name: typing.Any = field(...)`` if you don't want to set a
type.
.. warning::
For features that use the attribute name to create decorators
(for example, :ref:`validators <validators>`), you still *must*
assign :func:`field` / `attr.ib` to them. Otherwise Python will
either not find the name or try to use the default value to
call, for example, ``validator`` on it.
Attributes annotated as `typing.ClassVar`, and attributes that are
neither annotated nor set to an `field()` are **ignored**.
these (dict[str, object]):
A dictionary of name to the (private) return value of `field()`
mappings. This is useful to avoid the definition of your attributes
within the class body because you can't (for example, if you want
to add ``__repr__`` methods to Django models) or don't want to.
If *these* is not `None`, *attrs* will *not* search the class body
for attributes and will *not* remove any attributes from it.
The order is deduced from the order of the attributes inside
*these*.
Arguably, this is a rather obscure feature.
.. versionadded:: 20.1.0
.. versionchanged:: 21.3.0 Converters are also run ``on_setattr``.
.. versionadded:: 22.2.0
*unsafe_hash* as an alias for *hash* (for :pep:`681` compliance).
.. versionchanged:: 24.1.0
Instances are not compared as tuples of attributes anymore, but using a
big ``and`` condition. This is faster and has more correct behavior for
uncomparable values like `math.nan`.
.. versionadded:: 24.1.0
If a class has an *inherited* classmethod called
``__attrs_init_subclass__``, it is executed after the class is created.
.. deprecated:: 24.1.0 *hash* is deprecated in favor of *unsafe_hash*.
.. versionadded:: 24.3.0
Unless already present, a ``__replace__`` method is automatically
created for `copy.replace` (Python 3.13+ only).
.. versionchanged:: 25.4.0
*kw_only* now only applies to attributes defined in the current class,
and respects attribute-level ``kw_only=False`` settings.
.. versionadded:: 25.4.0
Added *force_kw_only* to go back to the previous *kw_only* behavior.
.. note::
The main differences to the classic `attr.s` are:
- Automatically detect whether or not *auto_attribs* should be `True`
(c.f. *auto_attribs* parameter).
- Converters and validators run when attributes are set by default --
if *frozen* is `False`.
- *slots=True*
Usually, this has only upsides and few visible effects in everyday
programming. But it *can* lead to some surprising behaviors, so
please make sure to read :term:`slotted classes`.
- *auto_exc=True*
- *auto_detect=True*
- *order=False*
- *force_kw_only=False*
- Some options that were only relevant on Python 2 or were kept around
for backwards-compatibility have been removed.
"""
def do_it(cls, auto_attribs):
return attrs(
maybe_cls=cls,
these=these,
repr=repr,
hash=hash,
unsafe_hash=unsafe_hash,
init=init,
slots=slots,
frozen=frozen,
weakref_slot=weakref_slot,
str=str,
auto_attribs=auto_attribs,
kw_only=kw_only,
cache_hash=cache_hash,
auto_exc=auto_exc,
eq=eq,
order=order,
auto_detect=auto_detect,
collect_by_mro=True,
getstate_setstate=getstate_setstate,
on_setattr=on_setattr,
field_transformer=field_transformer,
match_args=match_args,
force_kw_only=force_kw_only,
)
def wrap(cls):
"""
Making this a wrapper ensures this code runs during class creation.
We also ensure that frozen-ness of classes is inherited.
"""
nonlocal frozen, on_setattr
had_on_setattr = on_setattr not in (None, setters.NO_OP)
# By default, mutable classes convert & validate on setattr.
if frozen is False and on_setattr is None:
on_setattr = _DEFAULT_ON_SETATTR
# However, if we subclass a frozen class, we inherit the immutability
# and disable on_setattr.
for base_cls in cls.__bases__:
if base_cls.__setattr__ is _frozen_setattrs:
if had_on_setattr:
msg = "Frozen classes can't use on_setattr (frozen-ness was inherited)."
raise ValueError(msg)
on_setattr = setters.NO_OP
break
if auto_attribs is not None:
return do_it(cls, auto_attribs)
try:
return do_it(cls, True)
except UnannotatedAttributeError:
return do_it(cls, False)
# maybe_cls's type depends on the usage of the decorator. It's a class
# if it's used as `@attrs` but `None` if used as `@attrs()`.
if maybe_cls is None:
return wrap
return wrap(maybe_cls)
mutable = define
frozen = partial(define, frozen=True, on_setattr=None)
def field(
*,
default=NOTHING,
validator=None,
repr=True,
hash=None,
init=True,
metadata=None,
type=None,
converter=None,
factory=None,
kw_only=None,
eq=None,
order=None,
on_setattr=None,
alias=None,
):
"""
Create a new :term:`field` / :term:`attribute` on a class.
.. warning::
Does **nothing** unless the class is also decorated with
`attrs.define` (or similar)!
Args:
default:
A value that is used if an *attrs*-generated ``__init__`` is used
and no value is passed while instantiating or the attribute is
excluded using ``init=False``.
If the value is an instance of `attrs.Factory`, its callable will
be used to construct a new value (useful for mutable data types
like lists or dicts).
If a default is not set (or set manually to `attrs.NOTHING`), a
value *must* be supplied when instantiating; otherwise a
`TypeError` will be raised.
.. seealso:: `defaults`
factory (~typing.Callable):
Syntactic sugar for ``default=attr.Factory(factory)``.
validator (~typing.Callable | list[~typing.Callable]):
Callable that is called by *attrs*-generated ``__init__`` methods
after the instance has been initialized. They receive the
initialized instance, the :func:`~attrs.Attribute`, and the passed
value.
The return value is *not* inspected so the validator has to throw
an exception itself.
If a `list` is passed, its items are treated as validators and must
all pass.
Validators can be globally disabled and re-enabled using
`attrs.validators.get_disabled` / `attrs.validators.set_disabled`.
The validator can also be set using decorator notation as shown
below.
.. seealso:: :ref:`validators`
repr (bool | ~typing.Callable):
Include this attribute in the generated ``__repr__`` method. If
True, include the attribute; if False, omit it. By default, the
built-in ``repr()`` function is used. To override how the attribute
value is formatted, pass a ``callable`` that takes a single value
and returns a string. Note that the resulting string is used as-is,
which means it will be used directly *instead* of calling
``repr()`` (the default).
eq (bool | ~typing.Callable):
If True (default), include this attribute in the generated
``__eq__`` and ``__ne__`` methods that check two instances for
equality. To override how the attribute value is compared, pass a
callable that takes a single value and returns the value to be
compared.
.. seealso:: `comparison`
order (bool | ~typing.Callable):
If True (default), include this attributes in the generated
``__lt__``, ``__le__``, ``__gt__`` and ``__ge__`` methods. To
override how the attribute value is ordered, pass a callable that
takes a single value and returns the value to be ordered.
.. seealso:: `comparison`
hash (bool | None):
Include this attribute in the generated ``__hash__`` method. If
None (default), mirror *eq*'s value. This is the correct behavior
according the Python spec. Setting this value to anything else
than None is *discouraged*.
.. seealso:: `hashing`
init (bool):
Include this attribute in the generated ``__init__`` method.
It is possible to set this to False and set a default value. In
that case this attributed is unconditionally initialized with the
specified default value or factory.
.. seealso:: `init`
converter (typing.Callable | Converter):
A callable that is called by *attrs*-generated ``__init__`` methods
to convert attribute's value to the desired format.
If a vanilla callable is passed, it is given the passed-in value as
the only positional argument. It is possible to receive additional
arguments by wrapping the callable in a `Converter`.
Either way, the returned value will be used as the new value of the
attribute. The value is converted before being passed to the
validator, if any.
.. seealso:: :ref:`converters`
metadata (dict | None):
An arbitrary mapping, to be used by third-party code.
.. seealso:: `extending-metadata`.
type (type):
The type of the attribute. Nowadays, the preferred method to
specify the type is using a variable annotation (see :pep:`526`).
This argument is provided for backwards-compatibility and for usage
with `make_class`. Regardless of the approach used, the type will
be stored on ``Attribute.type``.
Please note that *attrs* doesn't do anything with this metadata by
itself. You can use it as part of your own code or for `static type
checking <types>`.
kw_only (bool | None):
Make this attribute keyword-only in the generated ``__init__`` (if
*init* is False, this parameter is ignored). If None (default),
mirror the setting from `attrs.define`.
on_setattr (~typing.Callable | list[~typing.Callable] | None | ~typing.Literal[attrs.setters.NO_OP]):
Allows to overwrite the *on_setattr* setting from `attr.s`. If left
None, the *on_setattr* value from `attr.s` is used. Set to
`attrs.setters.NO_OP` to run **no** `setattr` hooks for this
attribute -- regardless of the setting in `define()`.
alias (str | None):
Override this attribute's parameter name in the generated
``__init__`` method. If left None, default to ``name`` stripped
of leading underscores. See `private-attributes`.
.. versionadded:: 20.1.0
.. versionchanged:: 21.1.0
*eq*, *order*, and *cmp* also accept a custom callable
.. versionadded:: 22.2.0 *alias*
.. versionadded:: 23.1.0
The *type* parameter has been re-added; mostly for `attrs.make_class`.
Please note that type checkers ignore this metadata.
.. versionchanged:: 25.4.0
*kw_only* can now be None, and its default is also changed from False to
None.
.. seealso::
`attr.ib`
"""
return attrib(
default=default,
validator=validator,
repr=repr,
hash=hash,
init=init,
metadata=metadata,
type=type,
converter=converter,
factory=factory,
kw_only=kw_only,
eq=eq,
order=order,
on_setattr=on_setattr,
alias=alias,
)
def asdict(inst, *, recurse=True, filter=None, value_serializer=None):
"""
Same as `attr.asdict`, except that collections types are always retained
and dict is always used as *dict_factory*.
.. versionadded:: 21.3.0
"""
return _asdict(
inst=inst,
recurse=recurse,
filter=filter,
value_serializer=value_serializer,
retain_collection_types=True,
)
def astuple(inst, *, recurse=True, filter=None):
"""
Same as `attr.astuple`, except that collections types are always retained
and `tuple` is always used as the *tuple_factory*.
.. versionadded:: 21.3.0
"""
return _astuple(
inst=inst, recurse=recurse, filter=filter, retain_collection_types=True
)
def inspect(cls):
"""
Inspect the class and return its effective build parameters.
Warning:
This feature is currently **experimental** and is not covered by our
strict backwards-compatibility guarantees.
Args:
cls: The *attrs*-decorated class to inspect.
Returns:
The effective build parameters of the class.
Raises:
NotAnAttrsClassError: If the class is not an *attrs*-decorated class.
.. versionadded:: 25.4.0
"""
try:
return cls.__dict__["__attrs_props__"]
except KeyError:
msg = f"{cls!r} is not an attrs-decorated class."
raise NotAnAttrsClassError(msg) from None

Some files were not shown because too many files have changed in this diff Show More